1//===- ACCCGToGPU.cpp - Lower acc.compute_region to gpu.launch ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers `acc.compute_region` to the GPU dialect. For host-side
10// kernels it wraps the region in `gpu.launch`; for specialized acc routines
11// already inside a `gpu.func`, the body is lowered in place without emitting
12// a launch.
13//
14// Overview:
15// ---------
16// `acc.compute_region` is the compute-body representation produced after
17// OpenACC compute constructs are decomposed and parallelism has been assigned.
18// This pass is the final ACC-to-GPU lowering step for that body: it converts
19// nested `scf.parallel` / `scf.for` loops marked with `acc.par_dims` into GPU
20// block and thread parallelism, materializes privatization and reductions for
21// the device, inserts synchronization where shared state is observed across
22// threads, and erases the ACC scaffolding (`acc.compute_region`,
23// `acc.par_width`).
24//
25// Transformations:
26// ----------------
27// 1. Launch creation: outside a `gpu.func`, each `acc.compute_region` becomes a
28// `gpu.launch` whose grid and block sizes come from `acc.par_width` launch
29// operands (defaulting to 1). Kernel/module name attributes are preserved.
30// Inside a `gpu.func` (specialized acc routine), no launch is emitted.
31//
32// 2. Parallel loops: `scf.parallel` with a single `acc.par_dims` entry is
33// mapped to the corresponding GPU dimension (`block_*` or `thread_*`).
34// Sequential dimensions remain as `scf.parallel`/`scf.for` loops in the
35// generated kernel body.
36//
37// 3. Privatization: `acc.privatize` / `acc.private_local` storage is
38// materialized as one of: a per-thread `memref.alloca` (thread-private
39// arrays within the stack budget), an `acc.gpu_shared_memory` buffer
40// (gang-/worker-private arrays that fit the shared-memory budget), or a
41// `memref.alloc` whose pointer is broadcast to the block through a small
42// shared-memory slot (the data lives in global memory; shared memory only
43// holds the broadcast pointer).
44//
45// 4. Predication: `acc.predicate_region` becomes `scf.if` guarded by active
46// thread/block indices derived from `acc.par_dims` and launch dimensions.
47//
48// 5. Reductions: `acc.reduction_*` ops are lowered to GPU reduction and
49// synchronization primitives according to each reduction's parallel
50// dimensions and accumulator storage class.
51//
52// Example:
53// --------
54// Before:
55// %c128 = arith.constant 128 : index
56// %tx = acc.par_width %c128 par_dim(#acc.par_dim<thread_x>)
57// acc.compute_region launch(%arg0 = %tx) {
58// %c0 = arith.constant 0 : index
59// %c1 = arith.constant 1 : index
60// scf.parallel (%iv) = (%c0) to (%c128) step (%c1) {
61// ...
62// scf.reduce
63// } {acc.par_dims = #acc<par_dims[thread_x]>}
64// acc.yield
65// } <{origin = "acc.parallel"}>
66//
67// After:
68// gpu.launch blocks(%bidx, %bidy, %bidz) in (%gdimx = %c1, ...)
69// threads(%tidx, %tidy, %tidz) in (%bdimx = %c128, ...) {
70// ...
71// }
72//
73// Requirements:
74// -------------
75// - Must run on a GPU device type (`device-type` option); host and multicore
76// targets are rejected.
77// - Input must already be in the `acc.compute_region` form: nested SCF loops
78// carry `acc.par_dims`, privatization is expressed via `acc.privatize` /
79// `acc.private_local`, and reductions use the `acc.reduction_*` ops.
80// - Each `scf.parallel` processed by this pass is expected to have exactly
81// one parallel dimension and one induction variable.
82// - For acc routines, the `acc.compute_region` must live inside a `gpu.func`
83// in the GPU module.
84// - Uses `acc::OpenACCSupport` for NYI reporting and compiler remarks.
85// - Pass options: `max-workgroup-shared-memory`, `max-thread-private-stack`,
86// and `subgroup-size` (used for reductions and block-dimension alignment).
87//
88//===----------------------------------------------------------------------===//
89
90#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
91#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
92
93#include "mlir/Dialect/Arith/IR/Arith.h"
94#include "mlir/Dialect/Arith/Utils/Utils.h"
95#include "mlir/Dialect/Complex/IR/Complex.h"
96#include "mlir/Dialect/Func/IR/FuncOps.h"
97#include "mlir/Dialect/GPU/IR/GPUDialect.h"
98#include "mlir/Dialect/GPU/Utils/GPUUtils.h"
99#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
100#include "mlir/Dialect/MemRef/IR/MemRef.h"
101#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
102#include "mlir/Dialect/OpenACC/OpenACC.h"
103#include "mlir/Dialect/OpenACC/OpenACCParMapping.h"
104#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
105#include "mlir/Dialect/OpenACC/OpenACCUtilsGPU.h"
106#include "mlir/Dialect/OpenACC/OpenACCUtilsReduction.h"
107#include "mlir/Dialect/OpenACC/OpenACCUtilsType.h"
108#include "mlir/Dialect/SCF/IR/SCF.h"
109#include "mlir/IR/Block.h"
110#include "mlir/IR/BuiltinAttributeInterfaces.h"
111#include "mlir/IR/BuiltinTypes.h"
112#include "mlir/IR/Diagnostics.h"
113#include "mlir/IR/Dominance.h"
114#include "mlir/IR/IRMapping.h"
115#include "mlir/IR/OpDefinition.h"
116#include "mlir/IR/PatternMatch.h"
117#include "mlir/IR/SymbolTable.h"
118#include "mlir/IR/Value.h"
119#include "mlir/Interfaces/CallInterfaces.h"
120#include "mlir/Interfaces/SideEffectInterfaces.h"
121#include "mlir/Interfaces/ViewLikeInterface.h"
122#include "mlir/Support/LLVM.h"
123#include "mlir/Transforms/DialectConversion.h"
124#include "llvm/ADT/ArrayRef.h"
125#include "llvm/ADT/DenseMap.h"
126#include "llvm/ADT/STLExtras.h"
127#include "llvm/ADT/StringExtras.h"
128#include "llvm/ADT/Twine.h"
129#include "llvm/Support/Debug.h"
130#include <algorithm>
131#include <optional>
132#include <utility>
133
134namespace mlir {
135namespace acc {
136#define GEN_PASS_DEF_ACCCGTOGPU
137#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
138} // namespace acc
139} // namespace mlir
140
141#define DEBUG_TYPE "acc-cg-to-gpu"
142
143namespace {
144using namespace mlir;
145using namespace mlir::acc;
146
147enum class PrivateMemScope { Thread, Worker, Gang, None };
148
149/// Device label used in compiler remarks (e.g. "NVIDIA GPU").
150static std::string getDeviceRemarkQualifier(DeviceType deviceType) {
151 switch (deviceType) {
152 case DeviceType::None:
153 case DeviceType::Star:
154 case DeviceType::Default:
155 return "GPU";
156 default: {
157 std::string name;
158 llvm::StringRef deviceName = stringifyDeviceType(deviceType);
159 name.reserve(res_arg: deviceName.size());
160 for (char c : deviceName)
161 name.push_back(c: llvm::toUpper(x: c));
162 return name + " GPU";
163 }
164 }
165}
166
167/// True when \p op is inside a specialized acc routine function.
168static bool isInsideACCSpecializedRoutine(Operation *op) {
169 FunctionOpInterface funcOp = op->getParentOfType<FunctionOpInterface>();
170 return funcOp && acc::isSpecializedAccRoutine(op: funcOp);
171}
172
173/// Maps an acc.routine's parallelism clauses to a GPU parallel dimension.
174static GPUParallelDimAttr
175getAccRoutineParDim(RoutineOp routineOp, MLIRContext *ctx,
176 const ACCToGPUMappingPolicy &policy) {
177 if (routineOp.getGangDimValue() ||
178 routineOp.getGangDimValue(deviceType: DeviceType::Nvidia)) {
179 int64_t gangDimValue = routineOp.getGangDimValue(deviceType: DeviceType::Nvidia)
180 ? *routineOp.getGangDimValue(deviceType: DeviceType::Nvidia)
181 : *routineOp.getGangDimValue();
182 ParLevel gangLevel = getGangParLevel(gangDimValue);
183 return policy.gangDim(ctx, level: gangLevel);
184 }
185 if (routineOp.hasGang() || routineOp.hasGang(deviceType: DeviceType::Nvidia))
186 return policy.gangDim(ctx, level: ParLevel::gang_dim1);
187 if (routineOp.hasWorker() || routineOp.hasWorker(deviceType: DeviceType::Nvidia))
188 return policy.workerDim(ctx);
189 if (routineOp.hasVector() || routineOp.hasVector(deviceType: DeviceType::Nvidia))
190 return policy.vectorDim(ctx);
191 return policy.seqDim(ctx);
192}
193
194/// Looks up the acc.routine symbol associated with \p funcOp.
195static RoutineOp getRoutineOpForAccRoutineFunction(FunctionOpInterface funcOp,
196 const SymbolTable &symTab) {
197 if (isSpecializedAccRoutine(op: funcOp)) {
198 SpecializedRoutineAttr attr =
199 funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>(
200 name: getSpecializedRoutineAttrName());
201 return symTab.lookup<RoutineOp>(name: attr.getRoutine().getLeafReference());
202 }
203 RoutineInfoAttr routineInfo =
204 funcOp->getDiscardableAttrOfType<RoutineInfoAttr>(
205 name: getRoutineInfoAttrName());
206 if (!routineInfo || routineInfo.getAccRoutines().empty())
207 return nullptr;
208 return symTab.lookup<RoutineOp>(
209 name: routineInfo.getAccRoutines().front().getLeafReference());
210}
211
212/// Returns the parallelism level of a specialized acc routine function.
213static GPUParallelDimAttr
214getSpecializedRoutineDim(FunctionOpInterface funcOp,
215 const ACCToGPUMappingPolicy &policy) {
216 SpecializedRoutineAttr specAttr =
217 funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>(
218 name: getSpecializedRoutineAttrName());
219 assert(specAttr && "expected specialized routine attribute");
220 return policy.map(ctx: funcOp->getContext(), level: specAttr.getLevel().getValue());
221}
222
223/// Returns the parallelism dimension of a callee acc routine, if any.
224static GPUParallelDimAttr
225getAccRoutineCallParDim(CallOpInterface callOp,
226 const ACCToGPUMappingPolicy &policy) {
227 std::optional<CallInterfaceCallable> callee = callOp.getCallableForCallee();
228 if (!callee)
229 return nullptr;
230 SymbolRefAttr calleeSymbolRef = dyn_cast<SymbolRefAttr>(Val&: *callee);
231 if (!calleeSymbolRef)
232 return nullptr;
233 ModuleOp moduleOp = callOp->getParentOfType<ModuleOp>();
234 if (!moduleOp)
235 return nullptr;
236
237 SymbolTable symTab(moduleOp);
238 FunctionOpInterface funcOp =
239 symTab.lookup<FunctionOpInterface>(name: calleeSymbolRef.getLeafReference());
240 if (!funcOp)
241 return nullptr;
242
243 if (isSpecializedAccRoutine(op: funcOp))
244 return getSpecializedRoutineDim(funcOp, policy);
245 if (RoutineOp routineOp = getRoutineOpForAccRoutineFunction(funcOp, symTab))
246 return getAccRoutineParDim(routineOp, ctx: funcOp.getContext(), policy);
247 return nullptr;
248}
249
250/// Collects parallel dimensions from enclosing loops and the compute region.
251static SmallVector<GPUParallelDimAttr> getAncestorParDims(Operation *op) {
252 SmallVector<GPUParallelDimAttr> parDimsArray;
253 ComputeRegionOp computeRegion = op->getParentOfType<ComputeRegionOp>();
254 assert(computeRegion && "missing enclosing acc.compute_region");
255 scf::ParallelOp parentLoop = op->getParentOfType<scf::ParallelOp>();
256 // True while parentLoop is the innermost parallel ancestor of op.
257 bool isInnermostParallelParent = true;
258 while (parentLoop) {
259 bool hasNonSeqParDim = false;
260 if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: parentLoop)) {
261 for (GPUParallelDimAttr parDim : parDimsAttr.getArray()) {
262 insertParDim(parDims&: parDimsArray, parDim);
263 if (!parDim.isSeq())
264 hasNonSeqParDim = true;
265 }
266 }
267 // Include launch dims for a block-redundant ancestor when:
268 // - it is itself worksharing (e.g. vector + gpu_block_redundant), or
269 // - it is the innermost parallel parent (sequential remnant after
270 // partition-kernel-loops), so the body is not predicated on blockIdx.
271 // Do not include them for an outer sequential block-redundant wrapper
272 // around nested gang/worker/vector worksharing: that widens gang-private
273 // storage to per-thread.
274 if (hasGPUBlockRedundantAttr(op: parentLoop) &&
275 (hasNonSeqParDim || isInnermostParallelParent))
276 for (GPUParallelDimAttr parDim : computeRegion.getLaunchParDims())
277 insertParDim(parDims&: parDimsArray, parDim);
278 isInnermostParallelParent = false;
279 parentLoop = parentLoop->getParentOfType<scf::ParallelOp>();
280 }
281
282 if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: computeRegion))
283 for (GPUParallelDimAttr parDim : parDimsAttr.getArray())
284 insertParDim(parDims&: parDimsArray, parDim);
285 return parDimsArray;
286}
287
288/// Strips index casts to reach the underlying defining value.
289static Value stripIndexCastsFromValue(Value x) {
290 Operation *op = x.getDefiningOp();
291 if (!op)
292 return x;
293 while (arith::IndexCastOp castOp = dyn_cast<arith::IndexCastOp>(Val: op)) {
294 op = castOp->getOperand(idx: 0).getDefiningOp();
295 if (!op)
296 return x;
297 }
298 return op->getResult(idx: 0);
299}
300
301/// Extracts a compile-time integer constant from \p x, when known.
302static FailureOr<int64_t> extractIntConst(Value x,
303 bool stripIndexCasts = false) {
304 if (stripIndexCasts)
305 x = stripIndexCastsFromValue(x);
306 Operation *op = x.getDefiningOp();
307 if (op) {
308 if (arith::ConstantIntOp constOp = dyn_cast<arith::ConstantIntOp>(Val: op)) {
309 assert(constOp.getType().getIntOrFloatBitWidth() <= 64);
310 return constOp.value();
311 }
312 if (arith::ConstantIndexOp constOp = dyn_cast<arith::ConstantIndexOp>(Val: op))
313 return constOp.value();
314 }
315 return failure();
316}
317
318/// True when \p x is a constant equal to \p y (modulo index casts).
319static bool sameEffectiveValue(Value x, int64_t y) {
320 x = stripIndexCastsFromValue(x);
321 FailureOr<int64_t> conX = extractIntConst(x);
322 if (failed(Result: conX))
323 return false;
324 return *conX == y;
325}
326
327/// Continues tracking a memref through view-like and partial-access ops.
328static bool getPassThroughResults(Operation *userOp, Value trackedOperand,
329 SmallVectorImpl<Value> &passThroughResults) {
330 if (ViewLikeOpInterface viewLikeOp = dyn_cast<ViewLikeOpInterface>(Val: userOp)) {
331 if (viewLikeOp.getViewSource() == trackedOperand) {
332 passThroughResults.push_back(Elt: viewLikeOp.getViewDest());
333 return true;
334 }
335 return false;
336 }
337
338 // Partial-entity accesses (e.g. array element or field access) forward the
339 // base entity through to their results, so treat them as pass-through when
340 // the base entity is the value being tracked.
341 if (acc::PartialEntityAccessOpInterface partialAccess =
342 dyn_cast<acc::PartialEntityAccessOpInterface>(Val: userOp)) {
343 if (partialAccess.getBaseEntity() == trackedOperand) {
344 passThroughResults.append(in_start: userOp->result_begin(), in_end: userOp->result_end());
345 return true;
346 }
347 return false;
348 }
349 return false;
350}
351
352/// Skips memref view/cast chains to reach the underlying buffer.
353static Value unwrapMemRefConversion(Value v) {
354 while (Operation *op = v.getDefiningOp()) {
355 if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: op)) {
356 if (isa<MemRefType>(Val: viewLike.getViewSource().getType()) ||
357 isa<MemRefType>(Val: viewLike.getViewDest().getType())) {
358 v = viewLike.getViewSource();
359 continue;
360 }
361 }
362 break;
363 }
364 return v;
365}
366
367/// Walks back from a memref use to its defining `acc.private_local`, if any,
368/// looking through view/cast ops.
369static acc::PrivateLocalOp getPrivateLocalForMemref(Value memref);
370
371/// Returns the dimensions that own \p privateLocal.
372static GPUParallelDimsAttr
373getPrivateParDims(acc::PrivateLocalOp privateLocal,
374 acc::ComputeRegionOp computeRegion);
375
376/// True when the storage backing \p privateLocal is thread_x-private. An
377/// unknown scope conservatively counts as per-thread.
378static bool storageHasThreadX(acc::PrivateLocalOp privateLocal,
379 acc::ComputeRegionOp computeRegion) {
380 GPUParallelDimsAttr dims = getPrivateParDims(privateLocal, computeRegion);
381 return !dims || llvm::any_of(Range: dims.getArray(), P: [](GPUParallelDimAttr d) {
382 return d.isThreadX();
383 });
384}
385
386/// Returns the sole user of \p v, or null if it has zero or multiple uses.
387static Operation *getOnlyUser(Value v) {
388 if (!v.hasOneUse())
389 return nullptr;
390 return *v.user_begin();
391}
392
393/// True when \p privatize is privatized at thread_x parallelism.
394static bool isThreadXPrivatize(PrivatizeOp privatize) {
395 if (GPUParallelDimsAttr parDimsAttr = privatize.getParDimsAttr())
396 return llvm::any_of(Range: parDimsAttr.getArray(),
397 P: [](GPUParallelDimAttr d) { return d.isThreadX(); });
398 return false;
399}
400
401/// True when \p block is nested in a region that only part of the workgroup
402/// enters.
403static bool isInsideConditional(Block *block) {
404 if (!block)
405 return false;
406 for (Operation *parent = block->getParentOp();
407 parent && !isa<gpu::LaunchOp, FunctionOpInterface>(Val: parent);
408 parent = parent->getParentOp())
409 if (isa<RegionBranchOpInterface>(Val: parent) &&
410 !isa<LoopLikeOpInterface, scf::ExecuteRegionOp>(Val: parent))
411 return true;
412 return false;
413}
414
415/// Emits a workgroup-wide GPU barrier.
416static void emitGPUBarrierWorkgroup(OpBuilder &builder, Location loc) {
417 gpu::BarrierOp::create(builder, location: loc);
418}
419
420/// Emits a subgroup-scoped GPU barrier.
421static void emitGPUBarrierSubgroup(OpBuilder &builder, Location loc) {
422 gpu::BarrierOp::create(builder, location: loc, /*address_spaces=*/ArrayAttr{},
423 /*named_barrier=*/Value{},
424 scope: gpu::BarrierScope::Subgroup);
425}
426
427/// Emits a barrier over the lanes of this thread's ThreadY row. \p rowWidth
428/// (blockDim.x) must divide \p subgroupSize so the rows tile a subgroup. A
429/// subgroup-wide barrier would also tie together the other rows sharing it.
430static void emitGPUBarrierRow(OpBuilder &builder, Location loc,
431 int64_t rowWidth, int64_t subgroupSize) {
432 assert(rowWidth > 0 && subgroupSize % rowWidth == 0 &&
433 "row must tile the subgroup");
434 Type i32Ty = builder.getI32Type();
435 int64_t rowsPerSubgroup = subgroupSize / rowWidth;
436
437 // laneBase = (tid.y % rowsPerSubgroup) * rowWidth. Both operands are powers
438 // of two here, since rowWidth divides the subgroup size.
439 Value threadY = gpu::ThreadIdOp::create(builder, location: loc, resultType: builder.getIndexType(),
440 dimension: gpu::Dimension::y);
441 Value threadY32 = arith::IndexCastOp::create(builder, location: loc, out: i32Ty, in: threadY);
442 Value rowMask =
443 arith::ConstantIntOp::create(builder, location: loc, type: i32Ty, value: rowsPerSubgroup - 1);
444 Value rowIndex = arith::AndIOp::create(builder, location: loc, lhs: threadY32, rhs: rowMask);
445 Value width = arith::ConstantIntOp::create(builder, location: loc, type: i32Ty, value: rowWidth);
446 Value laneBase = arith::MulIOp::create(builder, location: loc, lhs: rowIndex, rhs: width);
447
448 Value laneBits = arith::ConstantIntOp::create(builder, location: loc, type: i32Ty,
449 value: (int64_t{1} << rowWidth) - 1);
450 Value mask = arith::ShLIOp::create(builder, location: loc, lhs: laneBits, rhs: laneBase);
451 NVVM::SyncWarpOp::create(builder, location: loc, mask);
452}
453
454/// The storage that a set of memory accesses refers to, identified by the value
455/// that names it (see getStorageRoot). \c unknown is set when an access refers
456/// to storage that could not be identified, which then may alias anything.
457struct SharedStorage {
458 llvm::SmallPtrSet<Value, 8> roots;
459 bool unknown = false;
460
461 bool empty() const { return roots.empty() && !unknown; }
462 void insert(const SharedStorage &other) {
463 roots.insert(I: other.roots.begin(), E: other.roots.end());
464 unknown |= other.unknown;
465 }
466};
467
468/// Lowers a single `acc.compute_region` to GPU dialect IR.
469class ACCCGToGPULowering {
470public:
471 explicit ACCCGToGPULowering(acc::ComputeRegionOp computeRegion,
472 RewriterBase &rewriter,
473 acc::OpenACCSupport &accSupport,
474 const ACCCGToGPUOptions &options)
475 : rewriter(rewriter), computeRegion(computeRegion),
476 accSupport(accSupport), options(options),
477 sharedMemBudget(
478 options.maxWorkgroupSharedMemory,
479 sumExistingSharedMemoryBytes(region&: computeRegion.getRegion())) {}
480
481 /// Main entry point: emit launch (if needed) and lower the region body.
482 LogicalResult rewrite();
483
484 gpu::LaunchOp getLaunch() const { return launch; }
485
486 bool hasFailed = false;
487 bool insideAccumulateGridStride = false;
488 Value reductionSharedBuf;
489 // Reduction-accumulator slot (memref) -> the block-reduced value stored into
490 // it; lets a block combine use the register instead of reloading.
491 llvm::DenseMap<Value, Value> reductionAccumValue;
492 // Combine reloads recorded before accumulates are lowered, patched up after.
493 llvm::SmallVector<std::pair<Value, memref::LoadOp>> pendingCombineReloads;
494
495private:
496 /// Lower a parallel loop to the GPU dimension given by its `acc.par_dims`.
497 void processParallelOp(scf::ParallelOp parallelOp);
498 /// Lower a sequential loop, including any required post-loop barriers.
499 template <typename LoopOp>
500 void processSeqLoop(LoopOp loopOp);
501 /// Lower an `acc.predicate_region` to a predicated `scf.if`.
502 void processPredicateRegion(acc::PredicateRegionOp interOp);
503 /// Materialize storage for an `acc.private_local`.
504 void
505 processPrivateLocal(acc::PrivateLocalOp privateLocal,
506 std::optional<int64_t> sharedMemCopies = std::nullopt);
507 /// Lower an `acc.privatize` to device storage.
508 Value processPrivatize(acc::PrivatizeOp privatize);
509 /// Heap-allocate privatize storage and insert a matching dealloc.
510 Value allocatePrivatizeHeapStorage(acc::PrivatizeOp privatize,
511 MemRefType baseTy,
512 ValueRange mappedDynamicSizes);
513 /// Clone and lower an `scf.execute_region`.
514 void processExecuteRegion(scf::ExecuteRegionOp op);
515 /// Lower `acc.reduction_accumulate`.
516 void processAccumulateOp(acc::ReductionAccumulateOp op);
517 /// Lower `acc.reduction_accumulate_array`.
518 void processAccumulateArrayOp(acc::ReductionAccumulateArrayOp op);
519 /// Lower `acc.reduction_init`.
520 void processReductionOp(acc::ReductionInitOp op);
521 /// Lower `acc.reduction_combine`.
522 void processReductionCombineOp(acc::ReductionCombineOp op);
523 /// Lower `acc.reduction_combine_region`.
524 void processCombineRegionOp(acc::ReductionCombineRegionOp op);
525 /// Clone a leaf operation into the lowered region.
526 void processGenericOp(Operation *op);
527 /// Clone and recursively lower an operation with nested regions.
528 void processGenericOpWithRegions(Operation *op);
529 /// Dispatch lowering for one operation in the compute-region body.
530 void processOp(Operation *op);
531
532 /// Emit an atomic reduction update to \p memref.
533 void constructAtomicAccumulation(Location loc, Value memref,
534 ValueRange indices, Value input,
535 arith::AtomicRMWKind kind);
536
537 /// Map an ACC reduction operator to an atomic RMW kind.
538 FailureOr<arith::AtomicRMWKind> getReductionKind(acc::ReductionOperator redOp,
539 Type type, Location loc);
540
541 /// Split launch dimensions into those that execute \p op and those that do
542 /// not, for predication and barrier placement.
543 std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>,
544 SmallVector<mlir::acc::GPUParallelDimAttr>>
545 computeActiveAndInactiveParDims(Operation *op, Block *block);
546
547 /// Build a predicate that is true only on inactive parallel dimensions.
548 Value
549 emitPredicate(Location loc,
550 SmallVector<mlir::acc::GPUParallelDimAttr> &inactiveParDims);
551
552 /// True when \p privateLocal may be placed in shared memory; returns the
553 /// number of copies needed, or nullopt if ineligible.
554 std::optional<int64_t>
555 isEligibleForSharedMemory(acc::PrivateLocalOp privateLocal,
556 MemRefType baseTy);
557
558 /// Reserve \p bytes from the shared-memory budget.
559 bool tryAllocateSharedMemory(int64_t bytes);
560
561 /// Record \p privateLocal as given per-thread storage, for the closing
562 /// remark. Whether that storage lands in registers or in local memory is
563 /// decided later, so the remark reports the allocation, not the placement.
564 /// Privates with no recoverable name or compiler temps are skipped.
565 void recordThreadPrivate(acc::PrivateLocalOp privateLocal);
566
567 /// Record \p privatizeOp as global memory storage, for the closing remark.
568 void recordGlobalMemory(acc::PrivatizeOp privatizeOp);
569
570 /// Element size in bytes for \p elementType .
571 int64_t getElementSizeInBytes(Location loc, Type elementType) const;
572
573 /// True when a static privatization fits in the per-thread stack budget.
574 bool canUseStackAlloca(MemRefType baseTy, Location loc,
575 int64_t maxThreadPrivateStack) const;
576
577 /// Emit a barrier scoped to the parallel dimensions in \p parDimsAttr.
578 void createBarrier(Location loc, mlir::acc::GPUParallelDimsAttr parDimsAttr);
579
580 /// Emit a per-row (per-worker) barrier.
581 /// Runtime branch on blockDim.y == 1 (workgroup-wide); compile-time choice
582 /// between gpu.barrier scope<subgroup> (staticBlockDimX <= subgroupSize)
583 /// and a named gpu.barrier (staticBlockDimX > subgroupSize) with tid.y+1.
584 void createPerRowBarrier(Location loc);
585
586 /// Insert barriers after a sequential loop when shared private state must be
587 /// visible to later loops.
588 void createBarrierAfterSeqLoop(Operation *loopOp);
589
590 /// Flush any deferred post-loop barriers that precede \p beforeOp.
591 void flushDeferredBarriersBefore(Operation *beforeOp);
592
593 /// True when \p loopOp may write shared memory read by a later sibling loop.
594 bool mayWriteSharedMemory(Operation *loopOp);
595
596 /// True when the thread-partitioned loop \p loop runs inside a sequential
597 /// loop nest, writes memory that more than one thread of its row or block
598 /// can see, and is followed (in the same or a later iteration of the nest)
599 /// by code that may read that memory. The sequential nest is executed by
600 /// every thread, so those readers can run before the threads that execute
601 /// the loop iterations have stored their results unless the loop is followed
602 /// by a barrier.
603 bool needsBarrierAfterThreadLoop(scf::ParallelOp loop);
604
605 /// True when \p root may read (\p reads) or write (\p writes) memory that is
606 /// not private to a single thread. Storage allocated inside \p localScope is
607 /// private to it. With \p workerLevel, worker-private storage also counts as
608 /// private, because the lanes of a worker-partitioned loop do not share it
609 /// with other workers. Operations inside \p ignore are skipped. When
610 /// \p collect is given, the walk does not stop at the first access and
611 /// records the storage of every shared access in it. When \p filter is given,
612 /// accesses to storage known to differ from the storage in \p filter are
613 /// ignored.
614 bool mayAccessSharedMemory(Operation *root, bool reads, bool writes,
615 bool workerLevel, Operation *localScope,
616 Operation *ignore = nullptr,
617 SharedStorage *collect = nullptr,
618 const SharedStorage *filter = nullptr);
619
620 /// Parallelism scope (thread, worker, or gang) of a privatized variable.
621 PrivateMemScope getPrivateMemScope(acc::PrivatizeOp privatizeOp);
622
623 /// Parallelism scope of the private buffer backing \p memref.
624 PrivateMemScope getPrivateScopeForMemref(Value memref);
625
626 /// `acc.privatize` that materialized the private buffer for \p memref.
627 acc::PrivatizeOp getPrivatizeForMemref(Value memref);
628
629 /// Collect privatize ops of the shared private buffers stored to under \p
630 /// root, keeping the gang- and worker-private ones apart: the two scopes need
631 /// different barriers, so a store at one scope must not stand in for the
632 /// other.
633 void collectSharedPrivateStores(
634 Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes,
635 llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes);
636
637 /// True when \p memref is backed by one of \p storePrivatizes at \p
638 /// storeScope.
639 bool isMatchingPrivateMemref(
640 Value memref, PrivateMemScope storeScope,
641 const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
642
643 /// True when a parallel region under \p seqLoopOp, outside \p skipOp, uses a
644 /// private buffer at \p storeScope backed by one of \p storePrivatizes.
645 bool hasParallelPrivateReuse(
646 LoopLikeOpInterface seqLoopOp, Operation *skipOp,
647 PrivateMemScope storeScope,
648 const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes);
649
650 /// Whether a predicate region needs a barrier before stores that will be read
651 /// by a later parallel loop over the same private memory.
652 PrivateMemScope needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp);
653
654 /// Whether \p loopOp needs a barrier at the top of its body because a
655 /// gang-private slot it writes is reused by the next iteration.
656 bool needsInLoopReuseBarrier(Operation *loopOp);
657
658 /// Emit `gpu.all_reduce` for a reduction partial.
659 void createGPUAllReduceOp(Location loc, Value input, Value memref,
660 arith::AtomicRMWKind kind,
661 mlir::acc::GPUParallelDimsAttr parDimsAttr,
662 ValueRange indices = {},
663 bool isPerThreadPrivateTarget = false);
664
665 /// Finish lowering a deferred `acc.reduction_accumulate`.
666 void postprocessAccumulateOp(acc::ReductionAccumulateOp op);
667
668 /// Finish lowering reductions attached to a parallel loop.
669 void postprocessLoopReduction(scf::ParallelOp parLoop);
670
671 /// Populate block/thread id and grid/block dimension maps for device
672 /// routines.
673 static void
674 createForAllDimensions(RewriterBase &rewriter, Location loc,
675 llvm::DenseMap<gpu::Processor, Value> &ids,
676 llvm::DenseMap<gpu::Processor, Value> &dims) {
677 ids[gpu::Processor::BlockX] = gpu::BlockIdOp::create(
678 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
679 ids[gpu::Processor::BlockY] = gpu::BlockIdOp::create(
680 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
681 ids[gpu::Processor::BlockZ] = gpu::BlockIdOp::create(
682 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z);
683 ids[gpu::Processor::ThreadX] = gpu::ThreadIdOp::create(
684 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
685 ids[gpu::Processor::ThreadY] = gpu::ThreadIdOp::create(
686 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
687 ids[gpu::Processor::ThreadZ] = gpu::ThreadIdOp::create(
688 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z);
689 dims[gpu::Processor::BlockX] = gpu::GridDimOp::create(
690 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
691 dims[gpu::Processor::BlockY] = gpu::GridDimOp::create(
692 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
693 dims[gpu::Processor::BlockZ] = gpu::GridDimOp::create(
694 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z);
695 dims[gpu::Processor::ThreadX] = gpu::BlockDimOp::create(
696 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
697 dims[gpu::Processor::ThreadY] = gpu::BlockDimOp::create(
698 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
699 dims[gpu::Processor::ThreadZ] = gpu::BlockDimOp::create(
700 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z);
701 }
702
703 /// Return the compute-region block argument for \p outside, adding an `ins`
704 /// operand when needed.
705 BlockArgument getOrAppendInsBlockArg(Value outside) {
706 if (std::optional<BlockArgument> blockArg =
707 computeRegion.getBlockArg(value: outside)) {
708 return *blockArg;
709 }
710 return computeRegion.appendInputArg(outside);
711 }
712
713 /// Wire dynamic privatization extents into the compute region as `ins` args.
714 void preparePrivatizeExtentInsOperands() {
715 computeRegion.walk(callback: [&](acc::PrivateLocalOp privateLocal) {
716 acc::PrivatizeOp privatizeOp =
717 getPrivatizeOp(privateLocal, computeRegion);
718 if (privatizeOp->getParentOfType<acc::ComputeRegionOp>() == computeRegion)
719 return;
720 for (Value extent : privatizeOp.getDynamicSizes())
721 getOrAppendInsBlockArg(outside: extent);
722 });
723 }
724
725 /// Resolve dynamic size operands for a privatized array.
726 SmallVector<Value>
727 resolvePrivateLocalDynamicExtents(acc::PrivateLocalOp privateLocal) {
728 acc::PrivatizeOp privatizeOp = getPrivatizeOp(privateLocal, computeRegion);
729 SmallVector<Value> extents;
730 for (Value extent : privatizeOp.getDynamicSizes()) {
731 if (std::optional<BlockArgument> blockArg =
732 computeRegion.getBlockArg(value: extent)) {
733 extents.push_back(Elt: mapping.lookupOrDefault(from: *blockArg));
734 continue;
735 }
736 extents.push_back(Elt: mapping.lookupOrDefault(from: extent));
737 }
738 return extents;
739 }
740
741 RewriterBase &rewriter;
742 acc::ComputeRegionOp computeRegion;
743
744 acc::OpenACCSupport &accSupport;
745 const ACCCGToGPUOptions &options;
746 gpu::LaunchOp launch;
747 IRMapping mapping;
748 llvm::SmallVector<scf::ParallelOp> loopReductions;
749 llvm::DenseMap<gpu::Processor, Value> threadIdMap;
750 llvm::DenseMap<gpu::Processor, Value> dimensionMap;
751 // True if ThreadY reduction exists, which triggers subgroup alignment
752 bool hasThreadYReduction = false;
753 // True if a ThreadX reduction exists, whose shuffles stay inside a row
754 bool hasThreadXReduction = false;
755 // True if an array reduction spans any thread dimension
756 bool hasThreadLevelArrayReduction = false;
757 // True if any ThreadX routine call exists in the kernel
758 bool hasThreadLevelRoutineCall = false;
759 // True when a per-row ThreadY barrier is emitted
760 bool hasThreadYBarrier = false;
761
762 // Reusable privatize broadcast slots per type; disabled for kernels.
763 llvm::DenseMap<Type, Value> privatizeBroadcastCache;
764
765 int64_t staticBlockDimX = 1024;
766 int64_t staticBlockDimY = 1024;
767 int64_t staticBlockDimZ = 1024;
768
769 /// True when the launch is (1, N, 1) with 1 < N <= subgroupSize, so every
770 /// worker owns one thread. N is capped by the worker-indexed shared buffers,
771 /// which hold subgroupSize entries.
772 bool isSingleThreadWorkerLaunch() const {
773 return staticBlockDimX == 1 && staticBlockDimZ == 1 &&
774 staticBlockDimY > 1 && staticBlockDimY <= options.subgroupSize;
775 }
776
777 /// True when only worker reductions would need the rows aligned. They combine
778 /// their partials in the lowest N threads of the block, not within a row, so
779 /// a row may sit anywhere in a subgroup. Array accumulates use atomics
780 /// instead and still need alignment.
781 bool isWorkerOnlyShuffleLaunch() const {
782 return hasThreadYReduction && !hasThreadXReduction &&
783 !hasThreadLevelArrayReduction && staticBlockDimZ == 1 &&
784 staticBlockDimY > 1 && staticBlockDimY <= options.subgroupSize &&
785 options.subgroupSize % staticBlockDimX == 0;
786 }
787
788 acc::DefaultACCToGPUMappingPolicy defaultPolicy;
789 SharedMemoryBudget sharedMemBudget;
790 SmallVector<std::string> sharedMemPrivateVarNames;
791 SmallVector<std::string> threadPrivateVarNames;
792 SmallVector<std::string> globalMemVarNames;
793 llvm::SmallVector<Operation *, 4> deferredBarrierSeqLoops;
794
795 Value getThreadId(Location loc, gpu::Dimension dim) {
796 return gpu::ThreadIdOp::create(builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: dim);
797 }
798
799 Value getBlockDim(Location loc, gpu::Dimension dim) {
800 return gpu::BlockDimOp::create(builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: dim);
801 }
802
803 /// Thread id for \p proc, from the launch op or the routine context map.
804 Value getGPUThreadIdFor(gpu::Processor proc) {
805 return getGPUThreadId(processor: proc, launch: getLaunch(), indexOps: threadIdMap);
806 }
807
808 /// Grid/block dimension for \p proc, from the launch op or routine map.
809 Value getGPUSizeFor(gpu::Processor proc) {
810 return getGPUSize(processor: proc, launch: getLaunch(), dimensionOps: dimensionMap);
811 }
812};
813
814int64_t ACCCGToGPULowering::getElementSizeInBytes(Location loc,
815 Type elementType) const {
816 ModuleOp module = computeRegion->getParentOfType<ModuleOp>();
817 if (std::optional<acc::TypeSizeAndAlignment> sizeAndAlignment =
818 accSupport.getTypeSizeAndAlignment(ty: elementType, module)) {
819 return sizeAndAlignment->first.getFixedValue();
820 }
821 std::string msg;
822 llvm::raw_string_ostream os(msg);
823 os << "element size computation for unsupported type: " << elementType;
824 (void)accSupport.emitNYI(loc, message: os.str());
825 return 0;
826}
827
828bool ACCCGToGPULowering::canUseStackAlloca(
829 MemRefType baseTy, Location loc, int64_t maxThreadPrivateStack) const {
830 for (int64_t dim : baseTy.getShape())
831 if (dim == ShapedType::kDynamic)
832 return false;
833 int64_t elementSize = getElementSizeInBytes(loc, elementType: baseTy.getElementType());
834 int64_t numElements = 1;
835 for (int64_t dim : baseTy.getShape()) {
836 if (numElements > maxThreadPrivateStack / std::max<int64_t>(a: dim, b: 1))
837 return false;
838 numElements *= dim;
839 }
840 return elementSize * numElements < maxThreadPrivateStack;
841}
842
843/// True if the accumulate spans a block dim or is nested in a block-mapped
844/// loop, i.e. each block owns the elements it reduces across threads. A
845/// thread-only accumulate with no block context grid-strides its element loop
846/// onto blocks, so per-thread partials would be dropped; such reductions must
847/// stay shared.
848static bool reductionHasBlockContext(acc::ReductionAccumulateArrayOp accArr) {
849 auto hasBlock = [](mlir::acc::GPUParallelDimsAttr parDims) {
850 return parDims && llvm::any_of(Range: parDims.getArray(),
851 P: [](auto pd) { return pd.isAnyBlock(); });
852 };
853 if (hasBlock(accArr.getParDimsAttr()))
854 return true;
855 for (scf::ParallelOp loop = accArr->getParentOfType<scf::ParallelOp>(); loop;
856 loop = loop->getParentOfType<scf::ParallelOp>()) {
857 if (hasBlock(mlir::acc::getParDimsAttr(op: loop)))
858 return true;
859 }
860 return false;
861}
862
863/// Returns the array reduction accumulate (through cast/view ops) that \p v
864/// feeds if it needs per-thread storage: its par_dims include a thread dim
865/// and it has block context so the cross-thread all_reduce is well defined.
866static acc::ReductionAccumulateArrayOp perThreadArrayReductionAccum(Value v) {
867 SmallVector<Value> worklist{v};
868 DenseSet<Value> seen;
869 while (!worklist.empty()) {
870 Value cur = worklist.pop_back_val();
871 if (!seen.insert(V: cur).second)
872 continue;
873 for (Operation *user : cur.getUsers()) {
874 if (acc::ReductionAccumulateArrayOp accArr =
875 dyn_cast<acc::ReductionAccumulateArrayOp>(Val: user)) {
876 bool hasThread = false;
877 for (auto pd : accArr.getParDims().getArray())
878 hasThread |= pd.isAnyThread();
879 if (hasThread && reductionHasBlockContext(accArr))
880 return accArr;
881 continue;
882 }
883 SmallVector<Value> through;
884 if (getPassThroughResults(userOp: user, trackedOperand: cur, passThroughResults&: through))
885 worklist.append(in_start: through.begin(), in_end: through.end());
886 else if (isa<ViewLikeOpInterface>(Val: user))
887 worklist.append(in_start: user->result_begin(), in_end: user->result_end());
888 }
889 }
890 return nullptr;
891}
892
893/// Store the reduction identity to every element of a freshly allocated
894/// per-thread array accumulator so all lanes start from identity (the original
895/// init loop may only run on one lane).
896static void initPerThreadArrayAccum(OpBuilder &b, Location loc, Value alloca,
897 MemRefType baseTy,
898 arith::AtomicRMWKind kind) {
899 assert(baseTy.getRank() > 0 && baseTy.hasStaticShape() &&
900 "per-thread array reduction accumulator must be static ranked");
901 Value ident = createIdentityValue(b, loc, type: baseTy.getElementType(), kind,
902 /*useOnlyFiniteValue=*/true);
903 Value lb = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0);
904 Value step = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 1);
905 SmallVector<Value> indices;
906 auto buildLoopNest = [&](auto &&self, unsigned dim) -> void {
907 if (dim == baseTy.getRank()) {
908 memref::StoreOp::create(builder&: b, location: loc, valueToStore: ident, memref: alloca, indices);
909 return;
910 }
911
912 Value ub = arith::ConstantIndexOp::create(builder&: b, location: loc, value: baseTy.getShape()[dim]);
913 auto forOp = scf::ForOp::create(builder&: b, location: loc, lowerBound: lb, upperBound: ub, step);
914 OpBuilder::InsertionGuard g(b);
915 b.setInsertionPoint(forOp.getBody()->getTerminator());
916 indices.push_back(Elt: forOp.getInductionVar());
917 self(self, dim + 1);
918 indices.pop_back();
919 };
920 buildLoopNest(buildLoopNest, 0);
921}
922
923std::optional<int64_t>
924ACCCGToGPULowering::isEligibleForSharedMemory(acc::PrivateLocalOp privateLocal,
925 MemRefType baseTy) {
926 // Cross-thread array reduction accumulators must stay per-thread when their
927 // storage scope includes thread_x. Gang-/worker-scoped array temps remain
928 // eligible for shared memory.
929 if (perThreadArrayReductionAccum(v: privateLocal.getResult()) &&
930 storageHasThreadX(privateLocal, computeRegion))
931 return std::nullopt;
932 ModuleOp module = computeRegion->getParentOfType<ModuleOp>();
933 FailureOr<bool> isCandidate = isPrivateLocalSharedMemoryCandidate(
934 privateLocal, computeRegion, module, policy: defaultPolicy, support: &accSupport);
935 if (failed(Result: isCandidate)) {
936 hasFailed = true;
937 return std::nullopt;
938 }
939 if (!isCandidate.value())
940 return std::nullopt;
941 std::optional<int64_t> upperBound =
942 getPrivateLocalSharedMemoryUpperBoundBytes(privateLocal, computeRegion,
943 module, policy: defaultPolicy);
944 assert(upperBound && "candidate private_local must have an upper bound");
945 int64_t elementSize =
946 getElementSizeInBytes(loc: privateLocal.getLoc(), elementType: baseTy.getElementType());
947 int64_t numElements = 1;
948 for (int64_t dim : baseTy.getShape())
949 numElements *= dim;
950 return *upperBound / (elementSize * numElements);
951}
952
953bool ACCCGToGPULowering::tryAllocateSharedMemory(int64_t bytes) {
954 return sharedMemBudget.tryAllocate(bytes);
955}
956
957void ACCCGToGPULowering::recordThreadPrivate(acc::PrivateLocalOp privateLocal) {
958 std::string varName = accSupport.getVariableName(v: privateLocal.getResult());
959 // A name that could not be recovered, or a compiler temp are skipped.
960 if (varName.empty())
961 return;
962 // The same variable reaches this point once per acc.private_local it has.
963 if (!llvm::is_contained(Range&: threadPrivateVarNames, Element: varName))
964 threadPrivateVarNames.push_back(Elt: varName);
965}
966
967void ACCCGToGPULowering::recordGlobalMemory(acc::PrivatizeOp privatizeOp) {
968 std::string varName = accSupport.getVariableName(v: privatizeOp.getResult());
969 // A name that could not be recovered, or a compiler temp are skipped.
970 if (varName.empty())
971 return;
972 // The same variable reaches this point once per acc.privatize it has.
973 if (!llvm::is_contained(Range&: globalMemVarNames, Element: varName))
974 globalMemVarNames.push_back(Elt: varName);
975}
976
977FailureOr<arith::AtomicRMWKind>
978ACCCGToGPULowering::getReductionKind(acc::ReductionOperator redOp, Type type,
979 Location loc) {
980 if (std::optional<arith::AtomicRMWKind> kind =
981 translateACCReductionOperator(redOp, type))
982 return *kind;
983
984 std::string msg;
985 llvm::raw_string_ostream os(msg);
986 os << "reduction operator (" << redOp << ") for type " << type;
987 (void)accSupport.emitNYI(loc, message: os.str());
988 return failure();
989}
990
991LogicalResult ACCCGToGPULowering::rewrite() {
992
993 // Pre-compute if thread-level reductions exist. ThreadY reduction generates
994 // shuffles which require subgroup alignment (blockDim.x = subgroupSize),
995 // meaning ThreadX lanes exist even without explicit ThreadX parallelism.
996 // ThreadX is tracked too, to tell worker-only reductions from row shuffles.
997 computeRegion->walk(callback: [&](acc::ReductionAccumulateOp op) {
998 for (auto parDim : op.getParDimsAttr().getArray()) {
999 hasThreadXReduction |= parDim.isThreadX();
1000 hasThreadYReduction |= parDim.isThreadY();
1001 }
1002 });
1003 computeRegion->walk(callback: [&](acc::ReductionAccumulateArrayOp op) {
1004 for (auto parDim : op.getParDimsAttr().getArray()) {
1005 if (!parDim.isAnyBlock())
1006 hasThreadLevelArrayReduction = true;
1007 }
1008 });
1009
1010 // Pre-compute if any thread-level (vector or worker) routine call exists.
1011 // Such routines partition work across ThreadX/ThreadY and emit workgroup-wide
1012 // barriers internally (e.g. for shared memory alloca synchronization), so all
1013 // workgroup threads must reach the call site for those barriers to converge.
1014 computeRegion->walk(callback: [&](CallOpInterface callOp) -> WalkResult {
1015 if (mlir::acc::GPUParallelDimAttr parDim =
1016 getAccRoutineCallParDim(callOp, policy: defaultPolicy)) {
1017 if (parDim.isThreadX() || parDim.isThreadY()) {
1018 hasThreadLevelRoutineCall = true;
1019 return WalkResult::interrupt();
1020 }
1021 }
1022 return WalkResult::advance();
1023 });
1024
1025 Location loc = computeRegion->getLoc();
1026 Value constantOne = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
1027
1028 auto launchArgument = [&](gpu::Processor processor) -> Value {
1029 mlir::acc::GPUParallelDimAttr parDim = mlir::acc::GPUParallelDimAttr::get(
1030 computeRegion->getContext(), processor);
1031 std::optional<Value> maybeLaunchArg =
1032 computeRegion.getKnownLaunchArg(parDim);
1033 LLVM_DEBUG(llvm::dbgs() << "ACCCGToGPU: launch-arg: "
1034 << " parDim: " << parDim << " gpu: " << processor
1035 << " widthValue: "
1036 << maybeLaunchArg.value_or(constantOne) << "\n");
1037
1038 return getValueOrCreateCastToIndexLike(
1039 b&: rewriter, loc, targetType: rewriter.getIndexType(),
1040 value: maybeLaunchArg.value_or(u&: constantOne));
1041 };
1042 LLVM_DEBUG(llvm::dbgs() << "ACCCGToGPU: creating gpu launch op: \n");
1043
1044 // acc.compute_region keeps launch argument as block argument, for rewriting
1045 // we now replace these with gpu.launch dimensions.
1046 auto mapLaunchArguments = [&](gpu::Processor processor, Value launchArg) {
1047 mlir::acc::GPUParallelDimAttr parDim = mlir::acc::GPUParallelDimAttr::get(
1048 computeRegion->getContext(), processor);
1049 std::optional<Value> kernelArg = computeRegion.getLaunchArg(parDim);
1050 if (kernelArg)
1051 mapping.map(from: computeRegion.gpuParWidth(processor), to: launchArg);
1052 };
1053
1054 llvm::StringRef blockDimXName = "blockDim.x";
1055 llvm::StringRef blockDimYName = "blockDim.y";
1056 std::string deviceLabel = getDeviceRemarkQualifier(deviceType: options.deviceType);
1057
1058 if (!computeRegion->getParentOfType<gpu::GPUFuncOp>()) {
1059 Value blockDimX = launchArgument(gpu::Processor::ThreadX);
1060 APInt bdxVal;
1061 if (matchPattern(value: blockDimX, pattern: m_ConstantInt(bind_value: &bdxVal)))
1062 staticBlockDimX = bdxVal.getSExtValue();
1063 Value blockDimY = launchArgument(gpu::Processor::ThreadY);
1064 APInt bdyVal;
1065 if (matchPattern(value: blockDimY, pattern: m_ConstantInt(bind_value: &bdyVal)))
1066 staticBlockDimY = bdyVal.getSExtValue();
1067 Value blockDimZ = launchArgument(gpu::Processor::ThreadZ);
1068 APInt bdzVal;
1069 if (matchPattern(value: blockDimZ, pattern: m_ConstantInt(bind_value: &bdzVal)))
1070 staticBlockDimZ = bdzVal.getSExtValue();
1071 Value gridDimX = launchArgument(gpu::Processor::BlockX);
1072 Value gridDimY = launchArgument(gpu::Processor::BlockY);
1073 Value gridDimZ = launchArgument(gpu::Processor::BlockZ);
1074
1075 // The format of the message is:
1076 // Generating [serial] {deviceLabel} code with gridDim=32x1x1
1077 // blockDim=256x1x1
1078 accSupport.emitRemark(op: computeRegion, messageFn: [&]() {
1079 auto getName = [&](Value val) -> std::string {
1080 // A negative width leaves the size to the launch.
1081 if (std::optional<int64_t> width = getConstantIntValue(ofr: val);
1082 width && *width < 0)
1083 return "(*)";
1084 std::string name = accSupport.getVariableName(v: val);
1085 return name.empty() ? "(*)" : name;
1086 };
1087 bool isEffectivelySerial =
1088 sameEffectiveValue(x: blockDimX, y: 1) &&
1089 sameEffectiveValue(x: blockDimY, y: 1) &&
1090 sameEffectiveValue(x: blockDimZ, y: 1) && sameEffectiveValue(x: gridDimX, y: 1) &&
1091 sameEffectiveValue(x: gridDimY, y: 1) && sameEffectiveValue(x: gridDimZ, y: 1);
1092 return (llvm::Twine("Generating ") +
1093 llvm::Twine(isEffectivelySerial ? "serial " : "") + deviceLabel +
1094 " code with gridDim=" + getName(gridDimX) + "x" +
1095 getName(gridDimY) + "x" + getName(gridDimZ) +
1096 " blockDim=" + getName(blockDimX) + "x" + getName(blockDimY) +
1097 "x" + getName(blockDimZ))
1098 .str();
1099 });
1100
1101 // Check if kernel has a stream operand for async execution
1102 if (mlir::Value streamValue = computeRegion.getStream()) {
1103 LLVM_DEBUG(llvm::dbgs()
1104 << "\nDEBUG: Creating async gpu.launch with stream: "
1105 << streamValue << "\n");
1106 launch = gpu::LaunchOp::create(
1107 builder&: rewriter, location: loc, gridSizeX: gridDimX, gridSizeY: gridDimY, gridSizeZ: gridDimZ, blockSizeX: blockDimX, blockSizeY: blockDimY,
1108 blockSizeZ: blockDimZ,
1109 /*dynamicSharedMemorySize=*/mlir::Value{},
1110 /*asyncTokenType=*/
1111 mlir::gpu::AsyncTokenType::get(ctx: rewriter.getContext()));
1112 // Add the stream as an async dependency
1113 launch.getAsyncDependenciesMutable().append(values: streamValue);
1114 } else {
1115 LLVM_DEBUG(llvm::dbgs()
1116 << "\nDEBUG: No stream, creating sync gpu.launch\n");
1117 launch = gpu::LaunchOp::create(builder&: rewriter, location: loc, gridSizeX: gridDimX, gridSizeY: gridDimY,
1118 gridSizeZ: gridDimZ, blockSizeX: blockDimX, blockSizeY: blockDimY, blockSizeZ: blockDimZ);
1119 }
1120
1121 // Transfer kernel function name and module name from acc.compute_region to
1122 // gpu.launch if present
1123 if (auto kernelFuncName = computeRegion.getKernelFuncNameAttr())
1124 launch.setFunctionAttr(kernelFuncName);
1125 if (auto kernelModuleName = computeRegion.getKernelModuleNameAttr())
1126 launch.setModuleAttr(kernelModuleName);
1127
1128 rewriter.setInsertionPointToEnd(&launch.getBody().front());
1129 gpu::TerminatorOp::create(builder&: rewriter, location: loc);
1130 rewriter.setInsertionPointToStart(&launch.getBody().front());
1131 mapLaunchArguments(gpu::Processor::BlockX,
1132 gpu::GridDimOp::create(builder&: rewriter, location: loc,
1133 resultType: rewriter.getIndexType(),
1134 dimension: gpu::Dimension::x));
1135 mapLaunchArguments(gpu::Processor::BlockY,
1136 gpu::GridDimOp::create(builder&: rewriter, location: loc,
1137 resultType: rewriter.getIndexType(),
1138 dimension: gpu::Dimension::y));
1139 mapLaunchArguments(gpu::Processor::BlockZ,
1140 gpu::GridDimOp::create(builder&: rewriter, location: loc,
1141 resultType: rewriter.getIndexType(),
1142 dimension: gpu::Dimension::z));
1143 mapLaunchArguments(gpu::Processor::ThreadX,
1144 gpu::BlockDimOp::create(builder&: rewriter, location: loc,
1145 resultType: rewriter.getIndexType(),
1146 dimension: gpu::Dimension::x));
1147 mapLaunchArguments(gpu::Processor::ThreadY,
1148 gpu::BlockDimOp::create(builder&: rewriter, location: loc,
1149 resultType: rewriter.getIndexType(),
1150 dimension: gpu::Dimension::y));
1151 mapLaunchArguments(gpu::Processor::ThreadZ,
1152 gpu::BlockDimOp::create(builder&: rewriter, location: loc,
1153 resultType: rewriter.getIndexType(),
1154 dimension: gpu::Dimension::z));
1155 } else {
1156 // Do not create gpu.launch for acc routine and map
1157 // to block/thread index and block/grid size instead
1158 // of launch arguments, using created maps.
1159 OpBuilder::InsertionGuard guard(rewriter);
1160 rewriter.setInsertionPointToStart(computeRegion->getBlock());
1161 createForAllDimensions(rewriter, loc, ids&: threadIdMap, dims&: dimensionMap);
1162 mapLaunchArguments(gpu::Processor::BlockX,
1163 dimensionMap[gpu::Processor::BlockX]);
1164 mapLaunchArguments(gpu::Processor::BlockY,
1165 dimensionMap[gpu::Processor::BlockY]);
1166 mapLaunchArguments(gpu::Processor::BlockZ,
1167 dimensionMap[gpu::Processor::BlockZ]);
1168 mapLaunchArguments(gpu::Processor::ThreadX,
1169 dimensionMap[gpu::Processor::ThreadX]);
1170 mapLaunchArguments(gpu::Processor::ThreadY,
1171 dimensionMap[gpu::Processor::ThreadY]);
1172 mapLaunchArguments(gpu::Processor::ThreadZ,
1173 dimensionMap[gpu::Processor::ThreadZ]);
1174 }
1175
1176 // Map input arguments for compute region; we go from an IsolatedFromAbove
1177 // operation to gpu.launch which is not IsolatedFromAbove.
1178 preparePrivatizeExtentInsOperands();
1179 Block *body = computeRegion.getBody();
1180 unsigned numLaunchArgs = computeRegion.getLaunchArgs().size();
1181 ValueRange inputArgs = computeRegion.getInputArgs();
1182 for (unsigned i = numLaunchArgs; i < body->getNumArguments(); ++i)
1183 mapping.map(from: body->getArgument(i), to: inputArgs[i - numLaunchArgs]);
1184
1185 assert(computeRegion.getRegion().hasOneBlock() &&
1186 "compute region only supports one block region for now");
1187 // process all operations inside kernel region
1188 for (auto &op : computeRegion.getRegion().getBlocks().front().getOperations())
1189 processOp(op: &op);
1190
1191 for (auto &parLoop : loopReductions)
1192 postprocessLoopReduction(parLoop);
1193
1194 // Replace combine reloads of a reduction slot with the block-reduced value.
1195 // Only when it dominates the reload; otherwise keep the reload.
1196 if (!pendingCombineReloads.empty() && launch) {
1197 DominanceInfo domInfo(launch);
1198 for (auto &[slot, loadOp] : pendingCombineReloads) {
1199 llvm::DenseMap<Value, Value>::iterator it =
1200 reductionAccumValue.find(Val: slot);
1201 if (it == reductionAccumValue.end())
1202 continue;
1203 if (!domInfo.dominates(a: it->second, b: loadOp.getOperation()))
1204 continue;
1205 rewriter.replaceOp(op: loadOp, newValues: ValueRange{it->second});
1206 }
1207 }
1208
1209 if (launch) {
1210 const int64_t subgroupSize = options.subgroupSize;
1211 const int64_t subgroupAlignMask = subgroupSize - 1;
1212
1213 // Adjust blockDim.x to be a multiple of subgroupSize. This is required
1214 // because:
1215 // - Subgroup reductions (gpu.all_reduce) require full subgroups
1216 // - Per-row workgroup barriers require blockDim.x aligned to subgroupSize
1217 // Tracked apart because only a ThreadX reduction shuffles within the row,
1218 // so only it needs the row to start at a subgroup boundary.
1219 bool needsThreadXAlign = false;
1220 bool needsThreadYAlign = false;
1221 bool alignThreadXReduction =
1222 getConstantIntValue(ofr: launch.getBlockSizeY()) != 1 ||
1223 getConstantIntValue(ofr: launch.getBlockSizeZ()) != 1;
1224
1225 // A reduction over both dimensions counts as a ThreadX one: it still
1226 // shuffles ThreadX lanes within the row.
1227 auto classifyAllReduce = [&](gpu::AllReduceOp allReduce) {
1228 bool hasThreadX = false;
1229 bool hasThreadY = false;
1230 for (auto parDim : mlir::acc::getParDimsAttr(op: allReduce).getArray()) {
1231 hasThreadX |= parDim.isThreadX();
1232 hasThreadY |= parDim.isThreadY();
1233 }
1234 if (hasThreadX && alignThreadXReduction)
1235 needsThreadXAlign = true;
1236 else if (hasThreadY)
1237 needsThreadYAlign = true;
1238 };
1239
1240 launch.walk(callback&: classifyAllReduce);
1241 // Also check if called routines have thread-level reductions.
1242 launch.walk(callback: [&](func::CallOp callOp) {
1243 if (gpu::GPUFuncOp callee =
1244 callOp->getParentOfType<ModuleOp>().lookupSymbol<gpu::GPUFuncOp>(
1245 name: callOp.getCallee()))
1246 callee.walk(callback&: classifyAllReduce);
1247 });
1248
1249 bool isShuffleEnabled = needsThreadXAlign || needsThreadYAlign;
1250
1251 std::optional<int64_t> constBlockDimX =
1252 getConstantIntValue(ofr: launch.getBlockSizeX());
1253 std::optional<int64_t> constBlockDimY =
1254 getConstantIntValue(ofr: launch.getBlockSizeY());
1255 std::optional<int64_t> constBlockDimZ =
1256 getConstantIntValue(ofr: launch.getBlockSizeZ());
1257
1258 // Skip subgroup alignment only when the total thread count is already
1259 // below a subgroup (constant blockDim.x in 2..subgroupSize-1 and
1260 // constant blockDim.y/z == 1). If blockDim.y/z > 1 or is unknown,
1261 // padding blockDim.x to a subgroup is still required so subgroups don't
1262 // cross row boundaries for row-local shuffle/ThreadY-barrier reductions.
1263 bool skipAlign = constBlockDimX && constBlockDimY && constBlockDimZ &&
1264 *constBlockDimX > 1 && *constBlockDimX < subgroupSize &&
1265 *constBlockDimY == 1 && *constBlockDimZ == 1;
1266
1267 // Padding a worker-only launch divides blockDim.y by the same factor and
1268 // leaves the folded-away workers as ThreadX lanes redoing each other's
1269 // work, so keep its shape instead.
1270 if (isSingleThreadWorkerLaunch())
1271 skipAlign = true;
1272 if (needsThreadYAlign && !needsThreadXAlign && !hasThreadYBarrier &&
1273 isWorkerOnlyShuffleLaunch())
1274 skipAlign = true;
1275
1276 if ((isShuffleEnabled || hasThreadYBarrier) && !skipAlign) {
1277 rewriter.setInsertionPoint(launch);
1278
1279 Value curBlockDimX = launch.getBlockSizeX();
1280 Value curBlockDimY = launch.getBlockSizeY();
1281 Value curBlockDimZ = launch.getBlockSizeZ();
1282
1283 // Emit a report on changing parallelism.
1284 accSupport.emitRemark(op: computeRegion, messageFn: [&]() {
1285 auto getName = [&](Value val) -> std::string {
1286 std::string name = accSupport.getVariableName(v: val);
1287 return name.empty() ? "(*)" : name;
1288 };
1289 std::string blockDimXValStr = getName(curBlockDimX);
1290 std::string blockDimYValStr = getName(curBlockDimY);
1291 llvm::StringRef kind =
1292 isShuffleEnabled ? "Shuffle reduction" : "ThreadY barrier";
1293 return (llvm::Twine(kind) +
1294 " is generated while adjusting the number of threads into "
1295 "groups of " +
1296 llvm::Twine(subgroupSize) + ".\n\t" + blockDimXName + ": `" +
1297 blockDimXValStr + "` to `((" + blockDimXValStr + " + " +
1298 llvm::Twine(subgroupAlignMask) + ") / " +
1299 llvm::Twine(subgroupSize) + ") * " + llvm::Twine(subgroupSize) +
1300 "`\n" + "\t" + blockDimYName + ": `" + blockDimYValStr +
1301 "` to `max(1, (new-" + blockDimXName + " * " + blockDimYValStr +
1302 ") / new-" + blockDimXName + ")`")
1303 .str();
1304 });
1305
1306 // Update the ThreadX length and the numbers of ThreadY and ThreadZ.
1307 // When the original block dimensions are compile-time
1308 // constants, compute the adjusted dimensions as constants directly so
1309 // that the GpuKernelOutliningPass can set `known_block_size` on the
1310 // outlined gpu.func.
1311 Value newBlockDimX, newBlockDimY, newBlockDimZ;
1312 if (constBlockDimX && constBlockDimY && constBlockDimZ) {
1313 int64_t bdx = *constBlockDimX;
1314 int64_t bdy = *constBlockDimY;
1315 int64_t bdz = *constBlockDimZ;
1316 int64_t alignedBdx =
1317 ((bdx + subgroupAlignMask) / subgroupSize) * subgroupSize;
1318 int64_t numXYThreads = bdx * bdy;
1319 int64_t numThreads = numXYThreads * bdz;
1320 int64_t newBdy = std::max<int64_t>(a: 1, b: numXYThreads / alignedBdx);
1321 int64_t newBdz =
1322 std::max<int64_t>(a: 1, b: numThreads / (alignedBdx * newBdy));
1323 newBlockDimX =
1324 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: alignedBdx);
1325 newBlockDimY = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: newBdy);
1326 newBlockDimZ = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: newBdz);
1327 } else {
1328 // numXYThreads = blockDim.x * blockDim.y
1329 Value numXYThreads =
1330 arith::MulIOp::create(builder&: rewriter, location: loc, lhs: curBlockDimX, rhs: curBlockDimY);
1331 Value numThreads =
1332 arith::MulIOp::create(builder&: rewriter, location: loc, lhs: numXYThreads, rhs: curBlockDimZ);
1333 // blockDim.x = ((blockDim.x + mask) / subgroupSize) * subgroupSize
1334 Value cstMask =
1335 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: subgroupAlignMask);
1336 Value cstSubgroupSize =
1337 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: subgroupSize);
1338 Value padded =
1339 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: curBlockDimX, rhs: cstMask);
1340 Value subgroupsRequired =
1341 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: padded, rhs: cstSubgroupSize);
1342 newBlockDimX = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: subgroupsRequired,
1343 rhs: cstSubgroupSize);
1344 // blockDim.y = max(1, numXYThreads / blockDim.x)
1345 Value quotient =
1346 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numXYThreads, rhs: newBlockDimX);
1347 Value cst1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
1348 newBlockDimY = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: cst1, rhs: quotient);
1349 // blockDim.z = max(1, numThreads / (blockDim.x * blockDim.y))
1350 Value newNumXYThreads =
1351 arith::MulIOp::create(builder&: rewriter, location: loc, lhs: newBlockDimX, rhs: newBlockDimY);
1352 quotient =
1353 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numThreads, rhs: newNumXYThreads);
1354 newBlockDimZ = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: cst1, rhs: quotient);
1355 }
1356
1357 launch.getBlockSizeXMutable().assign(value: newBlockDimX);
1358 launch.getBlockSizeYMutable().assign(value: newBlockDimY);
1359 launch.getBlockSizeZMutable().assign(value: newBlockDimZ);
1360 }
1361 }
1362
1363 if (hasFailed)
1364 return failure();
1365
1366 if (!threadPrivateVarNames.empty()) {
1367 accSupport.emitRemark(op: computeRegion, messageFn: [&]() {
1368 return (llvm::Twine("Local memory or registers used for ") +
1369 llvm::join(R&: threadPrivateVarNames, Separator: ","))
1370 .str();
1371 });
1372 }
1373
1374 if (!sharedMemPrivateVarNames.empty()) {
1375 accSupport.emitRemark(op: computeRegion, messageFn: [&]() {
1376 return (llvm::Twine("GPU shared memory used for ") +
1377 llvm::join(R&: sharedMemPrivateVarNames, Separator: ","))
1378 .str();
1379 });
1380 }
1381
1382 if (!globalMemVarNames.empty()) {
1383 accSupport.emitRemark(op: computeRegion, messageFn: [&]() {
1384 return (llvm::Twine("GPU global memory used for ") +
1385 llvm::join(R&: globalMemVarNames, Separator: ","))
1386 .str();
1387 });
1388 }
1389
1390 rewriter.eraseOp(op: computeRegion);
1391 return success();
1392}
1393
1394/// True when this accumulate is redundant in a nested reduction chain: the
1395/// value is a load of the destination memref and a sibling
1396/// acc.reduction_combine with block par_dims has already reduced %M across
1397/// threads in the block.
1398///
1399/// %v = memref.load %M[]
1400/// acc.reduction_accumulate %v to %M ...
1401/// acc.reduction_combine %M into %parent ... {block par_dims}
1402///
1403/// Lowering the accumulate again would double-count. Detection is structural;
1404/// nested reductions into per-thread privates do not match because their
1405/// combines are not block-scoped.
1406static bool isRedundantChainAccumulate(acc::ReductionAccumulateOp op) {
1407 Value memref = op.getMemref();
1408 memref::LoadOp loadOp = op.getValue().getDefiningOp<memref::LoadOp>();
1409 if (!loadOp || loadOp.getMemRef() != memref)
1410 return false;
1411 for (Operation *user : memref.getUsers()) {
1412 acc::ReductionCombineOp combineOp = dyn_cast<acc::ReductionCombineOp>(Val: user);
1413 if (!combineOp || combineOp.getDestMemref() != memref)
1414 continue;
1415 SmallVector<mlir::acc::GPUParallelDimAttr> parDims =
1416 getReductionCombineParDims(op: combineOp);
1417 if (llvm::any_of(Range&: parDims, P: [](mlir::acc::GPUParallelDimAttr d) {
1418 return d.isAnyBlock();
1419 })) {
1420 return true;
1421 }
1422 }
1423 return false;
1424}
1425
1426static GPUParallelDimsAttr
1427getPrivateParDims(acc::PrivateLocalOp privateLocal,
1428 acc::ComputeRegionOp computeRegion) {
1429 if (GPUParallelDimsAttr parDims = acc::getParDimsAttr(op: privateLocal))
1430 return parDims;
1431 if (acc::PrivatizeOp privatize = getPrivatizeOp(privateLocal, computeRegion))
1432 return privatize.getParDimsAttr();
1433 return {};
1434}
1435
1436/// True when \p privateLocal has one private slot per ThreadY row.
1437static bool isThreadYPrivate(acc::PrivateLocalOp privateLocal, bool allowBlock,
1438 acc::ComputeRegionOp computeRegion) {
1439 if (!privateLocal)
1440 return false;
1441 GPUParallelDimsAttr parDims = getPrivateParDims(privateLocal, computeRegion);
1442 if (!parDims)
1443 return false;
1444 return llvm::any_of(Range: parDims.getArray(),
1445 P: [](auto dim) { return dim.isThreadY(); }) &&
1446 llvm::all_of(Range: parDims.getArray(), P: [=](auto dim) {
1447 return dim.isThreadY() || (allowBlock && dim.isAnyBlock());
1448 });
1449}
1450
1451struct ThreadYBroadeningInfo {
1452 bool hasActiveWorkerCombine = false;
1453 bool hasExplicitInactiveCombine = false;
1454 bool hasBroadeningConflict = false;
1455 Operation *diagnosticOp = nullptr;
1456
1457 void merge(const ThreadYBroadeningInfo &other) {
1458 hasActiveWorkerCombine |= other.hasActiveWorkerCombine;
1459 hasExplicitInactiveCombine |= other.hasExplicitInactiveCombine;
1460 hasBroadeningConflict |= other.hasBroadeningConflict;
1461 if (!diagnosticOp)
1462 diagnosticOp = other.diagnosticOp;
1463 }
1464};
1465
1466/// True when executing \p op on additional ThreadY rows may change behavior.
1467static bool hasUnsafeEffectsWhenBroadening(Operation *op) {
1468 if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op)) {
1469 SmallVector<MemoryEffects::EffectInstance> effects;
1470 effectOp.getEffects(effects);
1471 return llvm::any_of(Range&: effects, P: [](const auto &effect) {
1472 return !isa<MemoryEffects::Read>(effect.getEffect());
1473 });
1474 }
1475 return !op->hasTrait<OpTrait::HasRecursiveMemoryEffects>();
1476}
1477
1478/// True when \p accumulator is the destination of a separate block-scoped
1479/// combine, so it holds a distinct per-worker partial rather than a broadcast.
1480static bool isFedByInnerBlockCombine(acc::PrivateLocalOp accumulator,
1481 Operation *selfCombine) {
1482 if (!accumulator)
1483 return false;
1484 for (Operation *user : accumulator.getResult().getUsers()) {
1485 if (user == selfCombine)
1486 continue;
1487 auto combineOp = dyn_cast<acc::ReductionCombineOp>(Val: user);
1488 if (!combineOp ||
1489 unwrapMemRefConversion(v: combineOp.getDestMemref()).getDefiningOp() !=
1490 accumulator.getOperation())
1491 continue;
1492 SmallVector<mlir::acc::GPUParallelDimAttr> parDims =
1493 getReductionCombineParDims(op: combineOp);
1494 if (llvm::any_of(Range&: parDims, P: [](mlir::acc::GPUParallelDimAttr d) {
1495 return d.isAnyBlock();
1496 }))
1497 return true;
1498 }
1499 return false;
1500}
1501
1502/// Records whether \p combineOp requires ThreadY to remain active.
1503static void classifyThreadYCombine(ThreadYBroadeningInfo &info,
1504 Operation *combineOp, Value src, Value dest,
1505 ArrayRef<GPUParallelDimAttr> parDims,
1506 acc::ComputeRegionOp computeRegion) {
1507 bool hasThreadY = llvm::any_of(
1508 Range&: parDims, P: [](GPUParallelDimAttr parDim) { return parDim.isThreadY(); });
1509 bool hasBlock = llvm::any_of(
1510 Range&: parDims, P: [](GPUParallelDimAttr parDim) { return parDim.isAnyBlock(); });
1511 acc::PrivateLocalOp srcPrivate =
1512 unwrapMemRefConversion(v: src).getDefiningOp<acc::PrivateLocalOp>();
1513 acc::PrivateLocalOp destPrivate =
1514 unwrapMemRefConversion(v: dest).getDefiningOp<acc::PrivateLocalOp>();
1515 bool hasPrivateDest = isa<acc::ReductionCombineOp>(Val: combineOp) && srcPrivate &&
1516 destPrivate &&
1517 getPrivatizeOp(privateLocal: srcPrivate, computeRegion) !=
1518 getPrivatizeOp(privateLocal: destPrivate, computeRegion);
1519 if (hasThreadY && hasBlock &&
1520 isThreadYPrivate(privateLocal: srcPrivate, allowBlock: hasPrivateDest, computeRegion)) {
1521 info.hasActiveWorkerCombine = true;
1522 return;
1523 }
1524
1525 // A block_y+thread_y accumulator fed by an inner block-scoped combine holds
1526 // a distinct partial per worker row, so its combine runs on every row; a
1527 // plain worker accumulate broadcasts via all_reduce and stays row-zero.
1528 if (hasThreadY && hasBlock &&
1529 isThreadYPrivate(privateLocal: srcPrivate, /*allowBlock=*/true, computeRegion) &&
1530 isFedByInnerBlockCombine(accumulator: srcPrivate, selfCombine: combineOp)) {
1531 info.hasActiveWorkerCombine = true;
1532 return;
1533 }
1534
1535 info.hasExplicitInactiveCombine = true;
1536 if (!info.diagnosticOp)
1537 info.diagnosticOp = combineOp;
1538}
1539
1540/// ThreadY must remain active for hierarchical combines over worker rows.
1541/// Broadening is rejected when siblings require inactive ThreadY or have side
1542/// effects; nested predicates enforce their own safety.
1543static ThreadYBroadeningInfo
1544analyzeThreadYBroadening(Block &predicateBlock,
1545 acc::ComputeRegionOp computeRegion) {
1546 ThreadYBroadeningInfo info;
1547 for (Operation &nestedOp : predicateBlock) {
1548 if (acc::PredicateRegionOp nestedPredicate =
1549 dyn_cast<acc::PredicateRegionOp>(Val&: nestedOp)) {
1550 ThreadYBroadeningInfo nestedInfo = analyzeThreadYBroadening(
1551 predicateBlock&: nestedPredicate.getRegion().front(), computeRegion);
1552 info.hasActiveWorkerCombine |= nestedInfo.hasActiveWorkerCombine;
1553 continue;
1554 }
1555 if (acc::ReductionCombineOp combineOp =
1556 dyn_cast<acc::ReductionCombineOp>(Val&: nestedOp)) {
1557 classifyThreadYCombine(
1558 info, combineOp, src: combineOp.getSrcMemref(), dest: combineOp.getDestMemref(),
1559 parDims: getReductionCombineParDims(op: combineOp), computeRegion);
1560 continue;
1561 }
1562 if (acc::ReductionCombineRegionOp combineRegionOp =
1563 dyn_cast<acc::ReductionCombineRegionOp>(Val&: nestedOp)) {
1564 classifyThreadYCombine(info, combineOp: combineRegionOp, src: combineRegionOp.getSrcVar(),
1565 dest: combineRegionOp.getDestVar(),
1566 parDims: getReductionCombineParDims(op: combineRegionOp),
1567 computeRegion);
1568 continue;
1569 }
1570 if (nestedOp.getNumRegions() != 0) {
1571 if (hasUnsafeEffectsWhenBroadening(op: &nestedOp)) {
1572 info.hasBroadeningConflict = true;
1573 if (!info.diagnosticOp)
1574 info.diagnosticOp = &nestedOp;
1575 }
1576 for (Region &region : nestedOp.getRegions())
1577 for (Block &nestedBlock : region)
1578 info.merge(other: analyzeThreadYBroadening(predicateBlock&: nestedBlock, computeRegion));
1579 continue;
1580 }
1581 if (hasUnsafeEffectsWhenBroadening(op: &nestedOp)) {
1582 info.hasBroadeningConflict = true;
1583 if (!info.diagnosticOp)
1584 info.diagnosticOp = &nestedOp;
1585 }
1586 }
1587 return info;
1588}
1589
1590std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>,
1591 SmallVector<mlir::acc::GPUParallelDimAttr>>
1592ACCCGToGPULowering::computeActiveAndInactiveParDims(Operation *op,
1593 Block *block) {
1594 MLIRContext *ctx = computeRegion->getContext();
1595
1596 mlir::acc::GPUParallelDimAttr routineParDim;
1597 if (isInsideACCSpecializedRoutine(op: computeRegion)) {
1598 FunctionOpInterface funcOp =
1599 computeRegion->getParentOfType<FunctionOpInterface>();
1600 routineParDim = getSpecializedRoutineDim(funcOp, policy: defaultPolicy);
1601 }
1602
1603 // The active set is precomputed separately from the ownership par_dims for
1604 // privatizations, and a predicate region may likewise state which dims run
1605 // it unpredicated; prefer that attribute when present. The inactive dims are
1606 // the launch dims which are not active.
1607 if (isa<acc::PrivateLocalOp, acc::PrivatizeOp, acc::PredicateRegionOp>(Val: op)) {
1608 if (mlir::acc::ActiveParDimsAttr precomputedActiveParDims =
1609 getActiveParDimsAttr(op)) {
1610 mlir::acc::GPUParallelDimAttr lowestParDim =
1611 mlir::acc::GPUParallelDimAttr::threadXDim(ctx);
1612
1613 SmallVector<mlir::acc::GPUParallelDimAttr> launchParDims;
1614 if (routineParDim) {
1615 for (mlir::acc::GPUParallelDimAttr parDim = routineParDim;
1616 parDim.getOrder() >= lowestParDim.getOrder();
1617 parDim = parDim.getOneLower()) {
1618 mlir::acc::insertParDim(parDims&: launchParDims, parDim);
1619 }
1620 } else {
1621 launchParDims = computeRegion.getLaunchParDims();
1622 }
1623
1624 SmallVector<mlir::acc::GPUParallelDimAttr> activeParDims(
1625 precomputedActiveParDims.getArray());
1626 SmallVector<mlir::acc::GPUParallelDimAttr> inactiveParDims;
1627 for (mlir::acc::GPUParallelDimAttr launchParDim : launchParDims) {
1628 if (launchParDim.getOrder() < lowestParDim.getOrder())
1629 break;
1630 if (!llvm::is_contained(Range&: activeParDims, Element: launchParDim))
1631 inactiveParDims.push_back(Elt: launchParDim);
1632 }
1633 return std::pair{activeParDims, inactiveParDims};
1634 }
1635 }
1636
1637 SmallVector<mlir::acc::GPUParallelDimAttr> ancestorParDims =
1638 getAncestorParDims(op);
1639 // Preserve whether there were any structural ancestor par-dims before
1640 // we start augmenting them based on inner uses (e.g. private_local).
1641 // This is needed for gang redundancy check - stores to worker-indexed
1642 // private_local should not disable redundant gang execution.
1643 bool noStructuralAncestorParDims =
1644 llvm::none_of(Range&: ancestorParDims, P: [](auto pd) { return !pd.isSeq(); });
1645
1646 if (routineParDim) {
1647 if (routineParDim.isThreadX()) {
1648 mlir::acc::insertParDim(parDims&: ancestorParDims,
1649 parDim: mlir::acc::GPUParallelDimAttr::threadYDim(ctx));
1650 }
1651 mlir::acc::insertParDim(parDims&: ancestorParDims,
1652 parDim: mlir::acc::GPUParallelDimAttr::blockXDim(ctx));
1653 }
1654
1655 // acc.private_local should use the same par_dims as acc.reduction_accumulate.
1656 if (acc::PrivateLocalOp privateLocalOp = dyn_cast<acc::PrivateLocalOp>(Val: op)) {
1657 for (Operation *user : privateLocalOp.getResult().getUsers()) {
1658 if (acc::ReductionAccumulateOp accumulateOp =
1659 dyn_cast<acc::ReductionAccumulateOp>(Val: user)) {
1660 if (accumulateOp.getMemref() == privateLocalOp.getResult()) {
1661 for (mlir::acc::GPUParallelDimAttr parDim :
1662 accumulateOp.getParDims().getArray()) {
1663 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1664 }
1665 }
1666 }
1667 // For decomposed complex reductions, the private_local is consumed
1668 // by an acc.reduction_combine{,_region} (no acc.reduction_accumulate
1669 // user). Mirror the par_dims so this private_local is treated as the
1670 // accumulator at the same parallelism level as a scalar reduction
1671 // would be (per-thread, not block-shared).
1672 if (acc::ReductionCombineOp combineOp =
1673 dyn_cast<acc::ReductionCombineOp>(Val: user)) {
1674 if (combineOp.getSrcMemref() == privateLocalOp.getResult()) {
1675 for (mlir::acc::GPUParallelDimAttr parDim :
1676 getReductionCombineParDims(op: combineOp)) {
1677 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1678 }
1679 }
1680 }
1681 if (auto combineRegionOp =
1682 dyn_cast<acc::ReductionCombineRegionOp>(Val: user)) {
1683 if (combineRegionOp.getSrcVar() == privateLocalOp.getResult()) {
1684 for (mlir::acc::GPUParallelDimAttr parDim :
1685 getReductionCombineParDims(op: combineRegionOp)) {
1686 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1687 }
1688 }
1689 }
1690 }
1691 // A dynamically-shaped privatization is materialized as a strided view
1692 // whose type does not reveal its parallel scope. Keep its own par_dims
1693 // active so the view is materialized (and predicated) per owning
1694 // thread/block. Statically-shaped privatizations are left to the stack-fit
1695 // decision so only do this for dynamic shapes.
1696 acc::PrivateType privTy =
1697 cast<acc::PrivateType>(Val: privateLocalOp.getPrivatized().getType());
1698 MemRefType baseTy = getPrivateBaseMemRefType(
1699 baseTy: privTy.getBaseTy(), module: computeRegion->getParentOfType<ModuleOp>());
1700 if (!baseTy.hasStaticShape()) {
1701 GPUParallelDimsAttr ownParDims =
1702 getPrivateParDims(privateLocal: privateLocalOp, computeRegion);
1703 if (ownParDims)
1704 for (GPUParallelDimAttr parDim : ownParDims.getArray())
1705 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1706 }
1707 }
1708
1709 bool hasBlock = false;
1710 for (mlir::acc::GPUParallelDimAttr parDim : ancestorParDims)
1711 if (parDim.isAnyBlock())
1712 hasBlock = true;
1713
1714 mlir::acc::GPUParallelDimAttr lowestParDim =
1715 mlir::acc::GPUParallelDimAttr::threadXDim(ctx);
1716 if (block) {
1717 ThreadYBroadeningInfo threadYInfo =
1718 analyzeThreadYBroadening(predicateBlock&: *block, computeRegion);
1719
1720 auto applyCombineParDims =
1721 [&](ArrayRef<mlir::acc::GPUParallelDimAttr> combineParDims) {
1722 for (mlir::acc::GPUParallelDimAttr parDim : combineParDims)
1723 mlir::acc::removeParDim(parDims&: ancestorParDims, parDim);
1724 return success();
1725 };
1726 block->walk(callback: [&](Operation *op) -> WalkResult {
1727 // Writes to acc.private_local must keep the privatization's par_dims
1728 // active so the write runs on all owning threads instead of being
1729 // predicated to a single lane.
1730 auto addPrivateStoreParDims = [&](Value target) {
1731 if (auto privateLocalOp = getPrivateLocalForMemref(memref: target)) {
1732 GPUParallelDimsAttr parDimsAttr =
1733 getPrivateParDims(privateLocal: privateLocalOp, computeRegion);
1734 if (parDimsAttr)
1735 for (auto parDim : parDimsAttr.getArray())
1736 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1737 }
1738 };
1739 if (auto memEffects = dyn_cast<MemoryEffectOpInterface>(Val: op)) {
1740 SmallVector<MemoryEffects::EffectInstance> effects;
1741 memEffects.getEffects(effects);
1742 for (const MemoryEffects::EffectInstance &effect : effects) {
1743 if (isa<MemoryEffects::Write>(Val: effect.getEffect()) &&
1744 effect.getValue())
1745 addPrivateStoreParDims(effect.getValue());
1746 }
1747 }
1748 // Consider ACC routine calls; routine calls should be predicated up to
1749 // one level above the parallel dimension of the callee.
1750 if (CallOpInterface callOp = dyn_cast<CallOpInterface>(Val: op)) {
1751 if (mlir::acc::GPUParallelDimAttr parDim =
1752 getAccRoutineCallParDim(callOp, policy: defaultPolicy)) {
1753 if (parDim.isBlockZ())
1754 lowestParDim = parDim;
1755 else
1756 lowestParDim = parDim.getOneHigher();
1757 }
1758 }
1759 // acc.reduction_combine_region should be predicated with the par_dims of
1760 // acc.reduction_accumulate. This is required when using combine between
1761 // kernel and loop in combined constructs.
1762 if (acc::ReductionCombineOp reductionCombineOp =
1763 dyn_cast<acc::ReductionCombineOp>(Val: op)) {
1764 if (failed(Result: applyCombineParDims(
1765 getReductionCombineParDims(op: reductionCombineOp))))
1766 return WalkResult::interrupt();
1767 }
1768 if (acc::ReductionCombineRegionOp combineRegionOp =
1769 dyn_cast<acc::ReductionCombineRegionOp>(Val: op)) {
1770 if (failed(Result: applyCombineParDims(
1771 getReductionCombineParDims(op: combineRegionOp))))
1772 return WalkResult::interrupt();
1773 }
1774 // An array accumulate reduces across its par_dims via gpu.all_reduce, so
1775 // all those threads must execute it - treat them as active (unlike the
1776 // scalar accumulate, which is active through its enclosing scf.parallel).
1777 if (acc::ReductionAccumulateArrayOp accArrayOp =
1778 dyn_cast<acc::ReductionAccumulateArrayOp>(Val: op)) {
1779 for (mlir::acc::GPUParallelDimAttr parDim :
1780 accArrayOp.getParDims().getArray()) {
1781 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim);
1782 }
1783 }
1784 return WalkResult::advance();
1785 });
1786 mlir::acc::GPUParallelDimAttr threadY =
1787 mlir::acc::GPUParallelDimAttr::threadYDim(ctx);
1788 bool baselineThreadYActive = llvm::is_contained(Range&: ancestorParDims, Element: threadY);
1789 if (threadYInfo.hasActiveWorkerCombine && !baselineThreadYActive) {
1790 if (threadYInfo.hasExplicitInactiveCombine ||
1791 threadYInfo.hasBroadeningConflict) {
1792 Operation *diagnosticOp =
1793 threadYInfo.diagnosticOp ? threadYInfo.diagnosticOp : op;
1794 (void)accSupport.emitNYI(
1795 loc: diagnosticOp->getLoc(),
1796 message: "operations in the same predicate region require incompatible "
1797 "ThreadY predication");
1798 hasFailed = true;
1799 return {};
1800 }
1801 mlir::acc::insertParDim(parDims&: ancestorParDims, parDim: threadY);
1802 }
1803 }
1804
1805 // Obtain launch dimensions
1806 SmallVector<mlir::acc::GPUParallelDimAttr> launchParDims;
1807 if (routineParDim) {
1808 for (mlir::acc::GPUParallelDimAttr parDim = routineParDim;
1809 parDim.getOrder() >= lowestParDim.getOrder();
1810 parDim = parDim.getOneLower()) {
1811 mlir::acc::insertParDim(parDims&: launchParDims, parDim);
1812 }
1813 } else {
1814 launchParDims = computeRegion.getLaunchParDims();
1815 }
1816
1817 // Compute dimensions that execute op
1818 SmallVector<mlir::acc::GPUParallelDimAttr> activeParDims, inactiveParDims;
1819 for (mlir::acc::GPUParallelDimAttr launchParDim : launchParDims) {
1820 if (launchParDim.getOrder() < lowestParDim.getOrder())
1821 break;
1822 if (llvm::find(Range&: ancestorParDims, Val: launchParDim) != ancestorParDims.end() ||
1823 (launchParDim.isAnyBlock() &&
1824 (noStructuralAncestorParDims || hasBlock))) {
1825 activeParDims.push_back(Elt: launchParDim);
1826 } else {
1827 inactiveParDims.push_back(Elt: launchParDim);
1828 }
1829 }
1830
1831 return std::pair{activeParDims, inactiveParDims};
1832}
1833
1834Value ACCCGToGPULowering::emitPredicate(
1835 Location loc, SmallVector<mlir::acc::GPUParallelDimAttr> &inactiveParDims) {
1836 Value predicate;
1837 for (mlir::acc::GPUParallelDimAttr inactiveParDim : inactiveParDims) {
1838 Value threadId = getGPUThreadIdFor(proc: inactiveParDim.getProcessor());
1839 TypedAttr zeroAttr = rewriter.getZeroAttr(type: threadId.getType());
1840 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, value: zeroAttr);
1841 Value cmp = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
1842 lhs: threadId, rhs: zero);
1843 if (predicate)
1844 predicate = arith::AndIOp::create(builder&: rewriter, location: loc, lhs: cmp, rhs: predicate);
1845 else
1846 predicate = cmp;
1847 }
1848 return predicate;
1849}
1850
1851void ACCCGToGPULowering::createBarrier(
1852 Location loc, mlir::acc::GPUParallelDimsAttr parDimsAttr) {
1853 bool hasAnyBlock = false, hasThreadY = false, hasThreadX = false;
1854 for (auto parDim : parDimsAttr.getArray()) {
1855 if (parDim.isAnyBlock())
1856 hasAnyBlock = true;
1857 if (parDim.isThreadY())
1858 hasThreadY = true;
1859 if (parDim.isThreadX())
1860 hasThreadX = true;
1861 }
1862
1863 if (hasAnyBlock || hasThreadY)
1864 emitGPUBarrierWorkgroup(builder&: rewriter, loc);
1865 else if (hasThreadX)
1866 createPerRowBarrier(loc);
1867}
1868
1869void ACCCGToGPULowering::createPerRowBarrier(Location loc) {
1870 // A row of a (1, N, 1) launch is a single thread; a subgroup barrier would
1871 // instead synchronize N workers.
1872 if (isSingleThreadWorkerLaunch())
1873 return;
1874
1875 // A worker-only row narrower than a subgroup covers only part of one, so
1876 // synchronize just its lanes. Both paths leave hasThreadYBarrier unset, so
1877 // the launch is not padded on account of this barrier.
1878 if (isWorkerOnlyShuffleLaunch() && staticBlockDimX < options.subgroupSize) {
1879 emitGPUBarrierRow(builder&: rewriter, loc, rowWidth: staticBlockDimX, subgroupSize: options.subgroupSize);
1880 return;
1881 }
1882
1883 hasThreadYBarrier = true;
1884
1885 if (staticBlockDimX <= options.subgroupSize) {
1886 emitGPUBarrierSubgroup(builder&: rewriter, loc);
1887 return;
1888 }
1889
1890 if (options.deviceType != mlir::acc::DeviceType::Nvidia) {
1891 (void)accSupport.emitNYI(
1892 loc,
1893 message: "per-row barrier to support worker parallelism on non-NVIDIA device");
1894 return;
1895 }
1896
1897 // Per-row barrier with fully runtime branching.
1898 // Three mutually exclusive paths:
1899 // blockDim.y == 1 -> gpu.barrier (workgroup-wide, only one worker)
1900 // blockDim.x <= subgroupSize -> gpu.barrier scope<subgroup>
1901 // blockDim.x > subgroupSize -> nvvm.barrier (tid.y + 1), blockDim.x
1902 // (named)
1903 //
1904 // Per-row barriers use tid.y+1 so IDs start at 1, avoiding clash with
1905 // barrier 0. When blockDim.y >= 16, worker 15's ID (16) wraps to
1906 // physical barrier 0; this is safe because named barriers are reusable
1907 // resources - workgroup-wide and per-row barriers on the same physical
1908 // barrier execute at different program points and never overlap.
1909 Value blockDimX = gpu::BlockDimOp::create(
1910 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
1911 Value blockDimY = gpu::BlockDimOp::create(
1912 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
1913 Value cst1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
1914 Value isSingleWorker = arith::CmpIOp::create(
1915 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: blockDimY, rhs: cst1);
1916
1917 auto outerIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isSingleWorker,
1918 /*withElseRegion=*/true);
1919
1920 // Then: blockDim.y == 1 -> workgroup-wide barrier (safe, only one worker)
1921 rewriter.setInsertionPointToStart(&outerIf.getThenRegion().front());
1922 emitGPUBarrierWorkgroup(builder&: rewriter, loc);
1923
1924 // Else: blockDim.y > 1 - choose between subgroup sync and named barrier
1925 rewriter.setInsertionPointToStart(&outerIf.getElseRegion().front());
1926 Value cstSubgroupSize =
1927 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: options.subgroupSize);
1928 Value isSubgroupSized = arith::CmpIOp::create(
1929 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ule, lhs: blockDimX, rhs: cstSubgroupSize);
1930
1931 auto innerIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isSubgroupSized,
1932 /*withElseRegion=*/true);
1933
1934 // Then: blockDim.x <= subgroupSize -> subgroup barrier (one worker per
1935 // subgroup)
1936 rewriter.setInsertionPointToStart(&innerIf.getThenRegion().front());
1937 emitGPUBarrierSubgroup(builder&: rewriter, loc);
1938
1939 // Else: blockDim.x > subgroupSize -> per-row named barrier with tid.y + 1.
1940 // The 1024-thread-per-block hardware limit with subgroup-aligned blockDim.x
1941 // (>= 64 here) guarantees blockDim.y <= 16, so IDs span at most 16
1942 // physical barriers (0-15) with no aliasing across workers.
1943 rewriter.setInsertionPointToStart(&innerIf.getElseRegion().front());
1944 Value threadYId = gpu::ThreadIdOp::create(
1945 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y);
1946 Value barrierId = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: threadYId, rhs: cst1);
1947 Type i32Ty = rewriter.getI32Type();
1948 Value barrierId32 =
1949 arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Ty, in: barrierId);
1950 Value numberOfThreads32 =
1951 arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Ty, in: blockDimX);
1952
1953 // GPU dialect named barriers do not have a means to create a custom barrier
1954 // id. Thus use nvvm directly.
1955 assert(options.deviceType == mlir::acc::DeviceType::Nvidia);
1956 NVVM::BarrierOp::create(builder&: rewriter, location: loc, barrierId: barrierId32, numberOfThreads: numberOfThreads32);
1957
1958 rewriter.setInsertionPointAfter(outerIf);
1959}
1960
1961/// Whether any later sibling of \p loopOp (or a loop nested inside one) is a
1962/// loop, i.e. whether some subsequent loop in the same region may read what
1963/// \p loopOp wrote. Used to skip a barrier after the last loop, where nothing
1964/// reads the data afterward.
1965static bool hasSubsequentLoopSibling(Operation *loopOp) {
1966 for (Operation *next = loopOp->getNextNode(); next;
1967 next = next->getNextNode()) {
1968 if (isa<scf::ParallelOp, scf::ForOp>(Val: next))
1969 return true;
1970 bool nested = false;
1971 next->walk(callback: [&](Operation *op) {
1972 if (isa<scf::ParallelOp, scf::ForOp>(Val: op)) {
1973 nested = true;
1974 return WalkResult::interrupt();
1975 }
1976 return WalkResult::advance();
1977 });
1978 if (nested)
1979 return true;
1980 }
1981 return false;
1982}
1983
1984/// Nearest enclosing sequential loop ancestor of \p op.
1985static LoopLikeOpInterface findFirstSequentialLoop(Operation *op) {
1986 auto isAllSequentialParDims = [](scf::ParallelOp par) -> bool {
1987 mlir::acc::GPUParallelDimsAttr pd = mlir::acc::getParDimsAttr(op: par);
1988 if (!pd || pd.getArray().empty())
1989 return false;
1990 return llvm::all_of(Range: pd.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) {
1991 return d.isSeq();
1992 });
1993 };
1994
1995 for (Operation *p = op->getParentOp(); p; p = p->getParentOp()) {
1996 // Do not need to check scf.for op's parents
1997 if (isa<scf::ForOp>(Val: p))
1998 return cast<LoopLikeOpInterface>(Val: p);
1999 if (scf::ParallelOp parOp = dyn_cast<scf::ParallelOp>(Val: p)) {
2000 if (isAllSequentialParDims(parOp))
2001 return cast<LoopLikeOpInterface>(Val: p);
2002 }
2003 }
2004 return nullptr;
2005}
2006
2007// A sequential loop that uses gang-private shared memory needs a
2008// workgroup-wide barrier afterward so every thread in the block observes the
2009// same state. When more work still lies between the loop and the next
2010// thread-reconvergence point in the same block, that barrier must follow that
2011// work; not sit immediately after the loop.
2012//
2013// The helpers below mark loop-body closure and other reconvergence points
2014// where any postponed barrier must be inserted.
2015
2016/// Marks the end of a loop body's iteration in the current block.
2017static bool isLoopBodyClosureOp(Operation *op) {
2018 return isa<scf::ReduceOp, scf::YieldOp, acc::YieldOp>(Val: op);
2019}
2020
2021/// Thread-reconvergence point where any postponed post-loop barrier for earlier
2022/// loops in this block must be inserted before proceeding.
2023static bool isDeferredBarrierFlushPoint(Operation *op) {
2024 if (isLoopBodyClosureOp(op))
2025 return true;
2026 // The next sequential loop may consume shared state produced by the prior
2027 // one.
2028 if (isa<scf::ForOp>(Val: op))
2029 return true;
2030 if (scf::ParallelOp parallelOp = dyn_cast<scf::ParallelOp>(Val: op)) {
2031 if (mlir::acc::hasParDimsAttr(op: parallelOp)) {
2032 if (mlir::acc::GPUParallelDimsAttr parDims =
2033 mlir::acc::getParDimsAttr(op: parallelOp)) {
2034 if (parDims.getArray().size() == 1 &&
2035 parDims.getArray().front().isSeq()) {
2036 return true;
2037 }
2038 }
2039 }
2040 }
2041 return false;
2042}
2043
2044/// True when a loop is followed by other work in the same block before the
2045/// loop body closes; the post-loop barrier must wait for that reconvergence
2046/// point instead of being placed right after the loop.
2047static bool hasTrailingSideEffectSiblings(Operation *loopOp) {
2048 for (Operation *next = loopOp->getNextNode(); next;
2049 next = next->getNextNode()) {
2050 return !isLoopBodyClosureOp(op: next);
2051 }
2052 return false;
2053}
2054
2055// Insert a barrier after a sequential loop when the surrounding parallel
2056// structure requires it, so that block threads observe the loop's writes to
2057// gang-/block-private shared memory before a later loop reads them. The barrier
2058// must land at an enclosing collective (block- or worker-level) point where all
2059// threads converge, so it cannot deadlock.
2060//
2061// The function walks the parent hierarchy of the loop to decide where (if at
2062// all) to place the barrier:
2063// - No enclosing parallel loop (top-level worksharing loop): emit a
2064// workgroup barrier when the loop writes shared memory and a later sibling
2065// loop may read it.
2066// - Nearest scf.parallel parent is itself sequential: a barrier *after* it
2067// would be unsafe when threads have varying iteration counts. However, when
2068// the loop has thread-level sub-loops collaborating on block-private memory
2069// (all threads participate every iteration), walk up to the block-level
2070// ancestor and insert the barrier there.
2071// - Nearest scf.parallel parent is a non-sequential parallel loop: walk up to
2072// the block-level (or worker/thread-y) ancestor and insert the barrier
2073// there, handling gang-redundant init loops and grid-stride remainders.
2074void ACCCGToGPULowering::createBarrierAfterSeqLoop(Operation *loopOp) {
2075 // Host loops enclosing the compute region are not GPU execution scopes.
2076 auto getParentParallel = [&](Operation *op) -> scf::ParallelOp {
2077 scf::ParallelOp parent = op->getParentOfType<scf::ParallelOp>();
2078 if (parent && computeRegion->isAncestor(other: parent))
2079 return parent;
2080 return {};
2081 };
2082 scf::ParallelOp wsLoop = getParentParallel(loopOp);
2083 if (!wsLoop) {
2084 // loopOp is a worksharing loop at the kernel-body top level (no enclosing
2085 // parallel loop). When it writes gang-private shared memory
2086 // that a later sibling loop reads, a workgroup barrier is needed between
2087 // them. The point just after a top-level loop is uniform (all threads reach
2088 // it), so a workgroup-wide barrier here cannot deadlock. Loops writing only
2089 // global/thread-private memory, or with no subsequent reader, get none.
2090 if (mayWriteSharedMemory(loopOp) && hasSubsequentLoopSibling(loopOp))
2091 emitGPUBarrierWorkgroup(builder&: rewriter, loc: loopOp->getLoc());
2092 return;
2093 }
2094
2095 mlir::acc::GPUParallelDimsAttr wsParDims = mlir::acc::getParDimsAttr(op: wsLoop);
2096 bool parentIsSeq =
2097 wsParDims.getArray().size() == 1 && wsParDims.getArray().front().isSeq();
2098
2099 if (parentIsSeq) {
2100 // loopOp is nested inside a sequential parent loop.
2101 // When the loop contains thread-level sub-loops,
2102 // multiple threads collaborate on shared (block-private) memory
2103 // within each iteration. Insert a block-level barrier.
2104 bool hasThreadSubLoop = false;
2105 loopOp->walk(callback: [&](scf::ParallelOp innerPar) -> WalkResult {
2106 if (innerPar.getOperation() == loopOp)
2107 return WalkResult::advance();
2108 mlir::acc::GPUParallelDimsAttr dims = mlir::acc::getParDimsAttr(op: innerPar);
2109 for (auto d : dims.getArray()) {
2110 if (d.isThreadX() || d.isThreadY()) {
2111 hasThreadSubLoop = true;
2112 return WalkResult::interrupt();
2113 }
2114 }
2115 return WalkResult::advance();
2116 });
2117 if (!hasThreadSubLoop)
2118 return;
2119 scf::ParallelOp threadLoop = getParentParallel(wsLoop);
2120 if (!threadLoop)
2121 return;
2122 scf::ParallelOp blockLoop = getParentParallel(threadLoop);
2123 if (!blockLoop)
2124 return;
2125 mlir::acc::GPUParallelDimsAttr parDimsAttr =
2126 mlir::acc::getParDimsAttr(op: blockLoop);
2127 if (parDimsAttr.hasOnlyBlockLevel())
2128 createBarrier(loc: loopOp->getLoc(), parDimsAttr);
2129 return;
2130 }
2131
2132 // Parent is a non-sequential parallel loop. Walk up to find the block-level
2133 // ancestor and insert a barrier there.
2134 scf::ParallelOp seqLoop = getParentParallel(wsLoop);
2135 if (!seqLoop) {
2136 // wsLoop is a worksharing loop directly under the compute region with no
2137 // gang ancestor: a gang-redundant init loop (e.g. a thread-level loop that
2138 // every gang runs to fill its own copy of gang-private shared memory). If
2139 // it writes gang-private (block-only) memory that a later sibling loop
2140 // reads, insert a workgroup barrier between them. The barrier sits just
2141 // after the top-level loop where all threads converge, so it cannot
2142 // deadlock.
2143 if (mayWriteSharedMemory(loopOp) && hasSubsequentLoopSibling(loopOp: wsLoop))
2144 emitGPUBarrierWorkgroup(builder&: rewriter, loc: loopOp->getLoc());
2145 return;
2146 }
2147 if (scf::ParallelOp outerParLoop = getParentParallel(seqLoop)) {
2148 mlir::acc::GPUParallelDimsAttr parDimsAttr =
2149 mlir::acc::getParDimsAttr(op: outerParLoop);
2150 if (parDimsAttr.hasOnlyBlockLevel()) {
2151 createBarrier(loc: loopOp->getLoc(), parDimsAttr);
2152 } else if (parDimsAttr.hasOnlyThreadYLevel()) {
2153 createPerRowBarrier(loc: loopOp->getLoc());
2154 } else if (parDimsAttr && parDimsAttr.isSeq()) {
2155 // outerParLoop is a sequential grid-stride remainder of a partitioned
2156 // gang loop, not the gang. Walk past the remainder(s) to the block-level
2157 // gang and barrier there.
2158 for (scf::ParallelOp gangLoop = getParentParallel(outerParLoop); gangLoop;
2159 gangLoop = getParentParallel(gangLoop)) {
2160 mlir::acc::GPUParallelDimsAttr gangDims =
2161 mlir::acc::getParDimsAttr(op: gangLoop);
2162 if (gangDims.hasOnlyBlockLevel()) {
2163 createBarrier(loc: loopOp->getLoc(), parDimsAttr: gangDims);
2164 break;
2165 }
2166 if (!gangDims.isSeq())
2167 break;
2168 }
2169 }
2170 return;
2171 }
2172
2173 // No block-level ancestor above seqLoop: the gang loop is seqLoop itself, a
2174 // gang(+vector) loop directly under the compute region. The fixed-depth
2175 // lookup above misses it (issue: a gang+vector init loop that writes
2176 // gang-private shared memory must be followed by a barrier before another
2177 // loop reads it). Only needed when the loop writes such shared memory; loops
2178 // writing global/thread-private memory (e.g. a plain gang-vector saxpy) do
2179 // not need one here.
2180 mlir::acc::GPUParallelDimsAttr parDimsAttr =
2181 mlir::acc::getParDimsAttr(op: seqLoop);
2182 if (parDimsAttr.hasOnlyBlockLevel() && mayWriteSharedMemory(loopOp)) {
2183 createBarrier(loc: loopOp->getLoc(), parDimsAttr);
2184 }
2185}
2186
2187bool ACCCGToGPULowering::mayWriteSharedMemory(Operation *loopOp) {
2188 bool found = false;
2189 loopOp->walk(callback: [&](memref::StoreOp storeOp) {
2190 // Trace the store target back to its backing acc.private_local through
2191 // view/cast/box ops, then check the privatization is gang-level.
2192 llvm::SmallVector<Value, 8> worklist{storeOp.getMemref()};
2193 llvm::SmallPtrSet<Value, 8> seen;
2194 while (!worklist.empty()) {
2195 Value v = worklist.pop_back_val();
2196 if (!seen.insert(Ptr: v).second)
2197 continue;
2198 Operation *def = v.getDefiningOp();
2199 if (!def)
2200 continue;
2201 if (acc::PrivateLocalOp privateLocal =
2202 dyn_cast<acc::PrivateLocalOp>(Val: def)) {
2203 acc::PrivatizeOp privatizeOp =
2204 getPrivatizeOp(privateLocal, computeRegion);
2205 // Only gang-level (block, no thread) private memory is a single copy
2206 // shared across the workgroup's threads, so a write needs a workgroup
2207 // barrier before another thread reads it. A [block_x, thread_x]
2208 // privatization is thread-private (one copy per thread); barriering on
2209 // it would deadlock when threads take different grid-stride iteration
2210 // counts.
2211 if (mlir::acc::GPUParallelDimsAttr parDims =
2212 privatizeOp.getParDimsAttr()) {
2213 bool hasBlock = false, hasThread = false;
2214 for (mlir::acc::GPUParallelDimAttr d : parDims.getArray()) {
2215 if (d.isAnyBlock())
2216 hasBlock = true;
2217 if (d.isThreadX() || d.isThreadY())
2218 hasThread = true;
2219 }
2220 if (hasBlock && !hasThread) {
2221 found = true;
2222 return WalkResult::interrupt();
2223 }
2224 }
2225 continue;
2226 }
2227 worklist.append(in_start: def->getOperands().begin(), in_end: def->getOperands().end());
2228 }
2229 return WalkResult::advance();
2230 });
2231 return found;
2232}
2233
2234/// True when \p memref is storage allocated inside \p scope, looking through
2235/// effect-free view and conversion ops. Such storage is created by the thread
2236/// that runs the code, so no other thread sees it.
2237static bool isAllocatedInside(Value memref, Operation *scope) {
2238 Value v = memref;
2239 while (Operation *def = v.getDefiningOp()) {
2240 if (!scope->isAncestor(other: def))
2241 return false;
2242 if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: def))
2243 if (effectOp.getEffectOnValue<MemoryEffects::Allocate>(value: v))
2244 return true;
2245 if (!isMemoryEffectFree(op: def) || def->getNumOperands() == 0)
2246 return false;
2247 if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: def))
2248 v = viewLike.getViewSource();
2249 else
2250 v = def->getOperand(idx: 0);
2251 }
2252 return false;
2253}
2254
2255/// True when \p op is a loop whose iterations run one after the other and are
2256/// repeated by every thread that reaches it.
2257static bool isRedundantSequentialLoop(Operation *op) {
2258 if (isa<scf::ForOp, scf::WhileOp>(Val: op))
2259 return true;
2260 scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: op);
2261 if (!par)
2262 return false;
2263 mlir::acc::GPUParallelDimsAttr dims = mlir::acc::getParDimsAttr(op: par);
2264 return dims && !dims.getArray().empty() &&
2265 llvm::all_of(Range: dims.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) {
2266 return d.isSeq();
2267 });
2268}
2269
2270/// The value that names the storage \p memref refers to, found by looking
2271/// through effect-free view and conversion ops, or null when the storage cannot
2272/// be identified. Two accesses with different non-null roots refer to
2273/// different storage. The arguments of a sequential loop are not identified:
2274/// they can name different storage in different iterations (a buffer swap).
2275/// An input of the compute region is traced into the value it is passed from,
2276/// so that inputs that are views of the same storage have the same root.
2277static Value getStorageRoot(Value memref) {
2278 // Ops such as fir.declare only have effects on resources that are not
2279 // memory (e.g. debug info), so they do not hide the storage behind them.
2280 auto onlyNonMemoryEffects = [](Operation *op) {
2281 MemoryEffectOpInterface effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op);
2282 if (!effectOp)
2283 return false;
2284 SmallVector<MemoryEffects::EffectInstance> effects;
2285 effectOp.getEffects(effects);
2286 return llvm::all_of(Range&: effects, P: [](const MemoryEffects::EffectInstance &e) {
2287 return e.getResource() != SideEffects::DefaultResource::get();
2288 });
2289 };
2290 Value v = memref;
2291 while (true) {
2292 Operation *def = v.getDefiningOp();
2293 if (!def) {
2294 Operation *owner = cast<BlockArgument>(Val&: v).getOwner()->getParentOp();
2295 if (isa<scf::ForOp, scf::WhileOp>(Val: owner))
2296 return Value();
2297 // The region cannot see where its inputs come from, but the values it is
2298 // passed may be views of the same storage. Keep the argument as the root
2299 // when that storage cannot be identified.
2300 if (auto computeOp = dyn_cast<acc::ComputeRegionOp>(Val: owner))
2301 if (Value operand = computeOp.getOperand(blockArg: cast<BlockArgument>(Val&: v)))
2302 if (Value outerRoot = getStorageRoot(memref: operand))
2303 return outerRoot;
2304 return v;
2305 }
2306 if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: def))
2307 if (effectOp.getEffectOnValue<MemoryEffects::Allocate>(value: v))
2308 return v;
2309 if ((!isMemoryEffectFree(op: def) && !onlyNonMemoryEffects(def)) ||
2310 def->getNumOperands() == 0)
2311 return Value();
2312 if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: def))
2313 v = viewLike.getViewSource();
2314 else
2315 v = def->getOperand(idx: 0);
2316 }
2317}
2318
2319bool ACCCGToGPULowering::mayAccessSharedMemory(Operation *root, bool reads,
2320 bool writes, bool workerLevel,
2321 Operation *localScope,
2322 Operation *ignore,
2323 SharedStorage *collect,
2324 const SharedStorage *filter) {
2325 bool found = false;
2326 // Reports a shared access to \p target (null when unknown). Without
2327 // \p collect the walk stops here; with it the storage is recorded and the
2328 // walk goes on.
2329 auto sharedAccess = [&](Value target) {
2330 Value storage = target ? getStorageRoot(memref: target) : Value();
2331 if (filter && storage && !filter->unknown &&
2332 !filter->roots.contains(Ptr: storage))
2333 return WalkResult::advance();
2334 found = true;
2335 if (!collect)
2336 return WalkResult::interrupt();
2337 if (storage)
2338 collect->roots.insert(Ptr: storage);
2339 else
2340 collect->unknown = true;
2341 return WalkResult::advance();
2342 };
2343 root->walk(callback: [&](Operation *op) {
2344 if (op->hasTrait<OpTrait::IsTerminator>())
2345 return WalkResult::advance();
2346 if (ignore && ignore->isAncestor(other: op))
2347 return WalkResult::advance();
2348 // Reductions combine their partial results with their own synchronization.
2349 if (isa<acc::ReductionAccumulateOp, acc::ReductionCombineOp,
2350 acc::ReductionCombineRegionOp>(Val: op))
2351 return WalkResult::advance();
2352 MemoryEffectOpInterface effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op);
2353 if (!effectOp) {
2354 // Ops that only contain other ops are covered by the walk. Anything else
2355 // with unknown effects (e.g. a call) may access shared memory.
2356 if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
2357 return WalkResult::advance();
2358 return sharedAccess(Value());
2359 }
2360 SmallVector<MemoryEffects::EffectInstance> effects;
2361 effectOp.getEffects(effects);
2362 for (const MemoryEffects::EffectInstance &effect : effects) {
2363 bool relevant =
2364 (writes && isa<MemoryEffects::Write>(Val: effect.getEffect())) ||
2365 (reads && isa<MemoryEffects::Read>(Val: effect.getEffect()));
2366 if (!relevant)
2367 continue;
2368 Value target = effect.getValue();
2369 if (target) {
2370 PrivateMemScope scope = getPrivateScopeForMemref(memref: target);
2371 if (scope == PrivateMemScope::Thread ||
2372 (workerLevel && scope == PrivateMemScope::Worker))
2373 continue;
2374 if (localScope && isAllocatedInside(memref: target, scope: localScope))
2375 continue;
2376 }
2377 if (sharedAccess(target).wasInterrupted())
2378 return WalkResult::interrupt();
2379 }
2380 return WalkResult::advance();
2381 });
2382 return found;
2383}
2384
2385bool ACCCGToGPULowering::needsBarrierAfterThreadLoop(scf::ParallelOp loop) {
2386 mlir::acc::GPUParallelDimsAttr loopDims = mlir::acc::getParDimsAttr(op: loop);
2387 if (!loopDims || loopDims.getArray().size() != 1)
2388 return false;
2389 mlir::acc::GPUParallelDimAttr loopDim = loopDims.getArray().front();
2390 if (!loopDim.isThreadX() && !loopDim.isThreadY())
2391 return false;
2392 bool workerLevel = loopDim.isThreadY();
2393
2394 // Inside a worker loop, createBarrierAfterSeqLoop already adds a per-row
2395 // barrier after the sequential remainder of each thread loop. Skip, to avoid
2396 // a repeat.
2397 for (Operation *parent = loop->getParentOp();
2398 parent && parent != computeRegion.getOperation();
2399 parent = parent->getParentOp()) {
2400 scf::ParallelOp parentLoop = dyn_cast<scf::ParallelOp>(Val: parent);
2401 if (!parentLoop)
2402 continue;
2403 mlir::acc::GPUParallelDimsAttr parentDims =
2404 mlir::acc::getParDimsAttr(op: parentLoop);
2405 if (!parentDims)
2406 continue;
2407 for (mlir::acc::GPUParallelDimAttr dim : parentDims.getArray())
2408 if (dim.isThreadY())
2409 return false;
2410 }
2411
2412 // A loop that works on a shared-memory tile made for it (for instance by the
2413 // cache directive lowering) fills, uses and writes back that tile with its
2414 // own synchronization.
2415 bool hasOwnTile = false;
2416 loop->walk(callback: [&](acc::GPUSharedMemoryOp) {
2417 hasOwnTile = true;
2418 return WalkResult::interrupt();
2419 });
2420 if (hasOwnTile)
2421 return false;
2422
2423 // Outside a sequential nest the loop is reached once, and what follows it is
2424 // handled by the predication and loop-level barriers.
2425 bool inSequentialNest = false;
2426 for (Operation *parent = loop->getParentOp();
2427 parent && parent != computeRegion.getOperation();
2428 parent = parent->getParentOp()) {
2429 if (isa<scf::ForOp, scf::WhileOp>(Val: parent)) {
2430 inSequentialNest = true;
2431 break;
2432 }
2433 }
2434 if (!inSequentialNest)
2435 return false;
2436
2437 // The iterations are spread over the threads, so a thread cannot rely on
2438 // what another thread did in the loop. Accesses to thread-private memory
2439 // never conflict. A later read conflicts with a write in the loop (read
2440 // after write) and a later write conflicts with a read (write after read) or
2441 // a write (write after write) in the loop, when both may refer to the same
2442 // storage.
2443 SharedStorage written, touched;
2444 mayAccessSharedMemory(root: loop, /*reads=*/false, /*writes=*/true, workerLevel,
2445 /*localScope=*/loop, /*ignore=*/nullptr, collect: &written);
2446 mayAccessSharedMemory(root: loop, /*reads=*/true, /*writes=*/false, workerLevel,
2447 /*localScope=*/loop, /*ignore=*/nullptr, collect: &touched);
2448 touched.insert(other: written);
2449 if (touched.empty())
2450 return false;
2451 bool loopWrites = !written.empty();
2452
2453 // True when \p op, executed after the loop, conflicts with it.
2454 auto conflictsWithLoop = [&](Operation *op, Operation *localScope,
2455 Operation *ignore) {
2456 if (loopWrites && mayAccessSharedMemory(
2457 root: op, /*reads=*/true, /*writes=*/false, workerLevel,
2458 localScope, ignore, /*collect=*/nullptr, filter: &written))
2459 return true;
2460 return mayAccessSharedMemory(root: op, /*reads=*/false, /*writes=*/true,
2461 workerLevel, localScope, ignore,
2462 /*collect=*/nullptr, filter: &touched);
2463 };
2464
2465 // Look for a conflicting access after the loop: later ops of every enclosing
2466 // block, and, for each enclosing loop, the ops in its body up to and
2467 // including the one that contains the loop, which run again in the next
2468 // iteration.
2469 for (Operation *cur = loop; cur && cur != computeRegion.getOperation();
2470 cur = cur->getParentOp()) {
2471 for (Operation *next = cur->getNextNode(); next; next = next->getNextNode())
2472 if (conflictsWithLoop(next, /*localScope=*/nullptr, /*ignore=*/nullptr))
2473 return true;
2474 Operation *parent = cur->getParentOp();
2475 if (!parent || parent == computeRegion.getOperation())
2476 break;
2477 if (!isRedundantSequentialLoop(op: parent))
2478 continue;
2479 // The condition and the other region of a while loop are not scanned.
2480 if (isa<scf::WhileOp>(Val: parent))
2481 return true;
2482 // Scan the body up to and including cur. Accesses inside cur itself, which
2483 // contains the loop, also run again in the next iteration. Storage
2484 // allocated inside cur is private to it.
2485 for (Operation *prev = &cur->getBlock()->front(); prev;
2486 prev = prev->getNextNode()) {
2487 if (prev != cur) {
2488 if (conflictsWithLoop(prev, /*localScope=*/nullptr, /*ignore=*/nullptr))
2489 return true;
2490 continue;
2491 }
2492 // The loop reads again in the next iteration what other threads wrote in
2493 // this one (e.g. a double-buffered stencil whose only body op is the
2494 // loop). Its own writes are not treated as conflicting with its writes in
2495 // the next iteration: the loop is assumed to store the same elements from
2496 // the same threads each time.
2497 if (loopWrites && mayAccessSharedMemory(
2498 root: loop, /*reads=*/true, /*writes=*/false, workerLevel,
2499 /*localScope=*/cur,
2500 /*ignore=*/nullptr, /*collect=*/nullptr, filter: &written))
2501 return true;
2502 // The other ops in cur run again too.
2503 if (cur != loop && conflictsWithLoop(cur, /*localScope=*/cur, loop))
2504 return true;
2505 break;
2506 }
2507 }
2508 return false;
2509}
2510
2511PrivateMemScope
2512ACCCGToGPULowering::getPrivateMemScope(acc::PrivatizeOp privatizeOp) {
2513 bool hasBlock = false;
2514 bool hasThreadX = false;
2515 bool hasThreadY = false;
2516 if (mlir::acc::GPUParallelDimsAttr parDims = privatizeOp.getParDimsAttr()) {
2517 for (mlir::acc::GPUParallelDimAttr d : parDims.getArray()) {
2518 if (d.isAnyBlock())
2519 hasBlock = true;
2520 if (d.isThreadX())
2521 hasThreadX = true;
2522 if (d.isThreadY())
2523 hasThreadY = true;
2524 }
2525 } else {
2526 for (mlir::acc::GPUParallelDimAttr d : computeRegion.getLaunchParDims())
2527 if (d.isAnyBlock())
2528 hasBlock = true;
2529 if (hasBlock)
2530 return PrivateMemScope::Gang;
2531 return PrivateMemScope::Thread;
2532 }
2533 if (hasThreadX)
2534 return PrivateMemScope::Thread;
2535 if (hasBlock && hasThreadY)
2536 return PrivateMemScope::Worker;
2537 if (hasBlock)
2538 return PrivateMemScope::Gang;
2539 return PrivateMemScope::Thread;
2540}
2541
2542/// Walks back from a memref use to its defining `acc.private_local`, if any.
2543static acc::PrivateLocalOp getPrivateLocalForMemref(Value memref) {
2544 llvm::SmallVector<Value, 8> worklist{memref};
2545 llvm::SmallPtrSet<Value, 8> seen;
2546 while (!worklist.empty()) {
2547 Value v = worklist.pop_back_val();
2548 if (!seen.insert(Ptr: v).second)
2549 continue;
2550 Operation *def = v.getDefiningOp();
2551 if (!def)
2552 continue;
2553 if (acc::PrivateLocalOp privateLocal = dyn_cast<acc::PrivateLocalOp>(Val: def))
2554 return privateLocal;
2555 worklist.append(in_start: def->getOperands().begin(), in_end: def->getOperands().end());
2556 }
2557 return nullptr;
2558}
2559
2560PrivateMemScope ACCCGToGPULowering::getPrivateScopeForMemref(Value memref) {
2561 if (auto privateLocal = getPrivateLocalForMemref(memref))
2562 return getPrivateMemScope(privatizeOp: getPrivatizeOp(privateLocal, computeRegion));
2563 return PrivateMemScope::None;
2564}
2565
2566acc::PrivatizeOp ACCCGToGPULowering::getPrivatizeForMemref(Value memref) {
2567 if (auto privateLocal = getPrivateLocalForMemref(memref))
2568 return getPrivatizeOp(privateLocal, computeRegion);
2569 return acc::PrivatizeOp();
2570}
2571
2572void ACCCGToGPULowering::collectSharedPrivateStores(
2573 Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes,
2574 llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes) {
2575 root->walk(callback: [&](memref::StoreOp storeOp) {
2576 PrivateMemScope scope = getPrivateScopeForMemref(memref: storeOp.getMemref());
2577 if (scope != PrivateMemScope::Gang && scope != PrivateMemScope::Worker)
2578 return;
2579 acc::PrivatizeOp privatize = getPrivatizeForMemref(memref: storeOp.getMemref());
2580 if (!privatize)
2581 return;
2582 if (scope == PrivateMemScope::Gang)
2583 gangPrivatizes.insert(Ptr: privatize.getOperation());
2584 else
2585 workerPrivatizes.insert(Ptr: privatize.getOperation());
2586 });
2587}
2588
2589bool ACCCGToGPULowering::isMatchingPrivateMemref(
2590 Value memref, PrivateMemScope storeScope,
2591 const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
2592 if (getPrivateScopeForMemref(memref) != storeScope)
2593 return false;
2594 acc::PrivatizeOp usePrivatize = getPrivatizeForMemref(memref);
2595 return usePrivatize && storePrivatizes.contains(Ptr: usePrivatize.getOperation());
2596}
2597
2598bool ACCCGToGPULowering::hasParallelPrivateReuse(
2599 LoopLikeOpInterface seqLoopOp, Operation *skipOp,
2600 PrivateMemScope storeScope,
2601 const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) {
2602 if (storePrivatizes.empty())
2603 return false;
2604
2605 bool hasParallelPrivateUse = false;
2606 seqLoopOp.getOperation()->walk(callback: [&](Operation *op) {
2607 // Ignore loads inside the predicate.
2608 if (skipOp->isAncestor(other: op))
2609 return WalkResult::advance();
2610
2611 Value memref;
2612 if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: op))
2613 memref = loadOp.getMemref();
2614 else if (memref::StoreOp storeOp = dyn_cast<memref::StoreOp>(Val: op))
2615 memref = storeOp.getMemref();
2616 else
2617 return WalkResult::advance();
2618
2619 if (!isMatchingPrivateMemref(memref, storeScope, storePrivatizes))
2620 return WalkResult::advance();
2621
2622 bool insideNestedParallel = false;
2623 for (Operation *p = op->getParentOp(); p && p != seqLoopOp.getOperation();
2624 p = p->getParentOp()) {
2625 if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: p)) {
2626 if (mlir::acc::GPUParallelDimsAttr pd =
2627 mlir::acc::getParDimsAttr(op: par)) {
2628 if (llvm::any_of(Range: pd.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) {
2629 return !d.isSeq();
2630 })) {
2631 insideNestedParallel = true;
2632 break;
2633 }
2634 }
2635 }
2636 }
2637 if (!insideNestedParallel)
2638 return WalkResult::advance();
2639 // An inner parallel region uses the same private memory.
2640 hasParallelPrivateUse = true;
2641 return WalkResult::interrupt();
2642 });
2643
2644 return hasParallelPrivateUse;
2645}
2646
2647PrivateMemScope
2648ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) {
2649
2650 // Check if we need a pre-predicate barrier first.
2651 // Next, check if the barrier should be gang- or worker-level.
2652 LoopLikeOpInterface seqLoopOp = findFirstSequentialLoop(op: interOp);
2653 if (!seqLoopOp)
2654 return PrivateMemScope::None;
2655
2656 // Check if any op in the predicate region stores to gang- or worker-private
2657 // memory. If not, no barrier is needed.
2658 llvm::SmallPtrSet<Operation *, 4> gangPrivatizes, workerPrivatizes;
2659 collectSharedPrivateStores(root: interOp, gangPrivatizes, workerPrivatizes);
2660
2661 // A reused gang slot needs the workgroup barrier, which also reconverges the
2662 // worker rows, so look for that hazard first. A gang store whose slot is
2663 // never read back does not stand in for a worker one: fall through and check
2664 // the worker slots on their own, otherwise their per-row barrier is lost.
2665 if (hasParallelPrivateReuse(seqLoopOp, skipOp: interOp, storeScope: PrivateMemScope::Gang,
2666 storePrivatizes: gangPrivatizes))
2667 return PrivateMemScope::Gang;
2668 if (hasParallelPrivateReuse(seqLoopOp, skipOp: interOp, storeScope: PrivateMemScope::Worker,
2669 storePrivatizes: workerPrivatizes))
2670 return PrivateMemScope::Worker;
2671 return PrivateMemScope::None;
2672}
2673
2674bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) {
2675 // A sequential scf.parallel is the remainder of a partitioned gang loop: the
2676 // gang iterations this workgroup still runs as a grid stride. Its scalar
2677 // stores stay predicated and already reconverge between iterations, so the
2678 // reuse hole is the nested scf.for coming from `acc loop seq`.
2679 if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: loopOp))
2680 return false;
2681
2682 // The hazard needs a worker- or vector-level routine call in the body: every
2683 // workgroup thread reaches such a call and the callee synchronizes
2684 // internally, so threads leave it in no particular order and can then be
2685 // running different iterations of this loop at the same time.
2686 if (!hasThreadLevelRoutineCall)
2687 return false;
2688 bool callsThreadLevelRoutine = false;
2689 loopOp->walk(callback: [&](CallOpInterface callOp) -> WalkResult {
2690 if (mlir::acc::GPUParallelDimAttr parDim =
2691 getAccRoutineCallParDim(callOp, policy: defaultPolicy)) {
2692 if (parDim.isThreadX() || parDim.isThreadY()) {
2693 callsThreadLevelRoutine = true;
2694 return WalkResult::interrupt();
2695 }
2696 }
2697 return WalkResult::advance();
2698 });
2699 if (!callsThreadLevelRoutine)
2700 return false;
2701
2702 // Only gang-level privatizations are a single copy shared by the whole
2703 // workgroup and reused by every iteration; worker- and thread-private copies
2704 // cannot be clobbered by another thread.
2705 llvm::SmallPtrSet<Operation *, 4> storedPrivatizes, workerPrivatizes;
2706 collectSharedPrivateStores(root: loopOp, gangPrivatizes&: storedPrivatizes, workerPrivatizes);
2707 if (storedPrivatizes.empty())
2708 return false;
2709
2710 // Storing alone is harmless: the next iteration only clobbers a value someone
2711 // still needs when the slot is read back in the body, either directly or by a
2712 // callee it is handed to.
2713 bool reusesGangPrivate = false;
2714 loopOp->walk(callback: [&](Operation *op) -> WalkResult {
2715 if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: op))
2716 reusesGangPrivate = isMatchingPrivateMemref(
2717 memref: loadOp.getMemref(), storeScope: PrivateMemScope::Gang, storePrivatizes: storedPrivatizes);
2718 else if (CallOpInterface callOp = dyn_cast<CallOpInterface>(Val: op))
2719 reusesGangPrivate = llvm::any_of(Range: callOp.getArgOperands(), P: [&](Value arg) {
2720 return isMatchingPrivateMemref(memref: arg, storeScope: PrivateMemScope::Gang,
2721 storePrivatizes: storedPrivatizes);
2722 });
2723 return reusesGangPrivate ? WalkResult::interrupt() : WalkResult::advance();
2724 });
2725 if (!reusesGangPrivate)
2726 return false;
2727
2728 // A barrier inside the body is only safe when the whole workgroup runs every
2729 // iteration, so bail out under any thread-level worksharing loop, whose trip
2730 // count varies per thread.
2731 for (Operation *p = loopOp->getParentOp();
2732 p && p != computeRegion.getOperation(); p = p->getParentOp()) {
2733 scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: p);
2734 if (!par)
2735 continue;
2736 if (mlir::acc::GPUParallelDimsAttr parDims = mlir::acc::getParDimsAttr(op: par))
2737 for (mlir::acc::GPUParallelDimAttr d : parDims.getArray())
2738 if (d.isThreadX() || d.isThreadY())
2739 return false;
2740 }
2741 return true;
2742}
2743
2744void ACCCGToGPULowering::processPredicateRegion(
2745 acc::PredicateRegionOp interOp) {
2746 LLVM_DEBUG(llvm::dbgs() << "processing predicate region: ";
2747 interOp->print(llvm::dbgs()); llvm::dbgs() << "\n");
2748 Location loc = interOp->getLoc();
2749
2750 std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>,
2751 SmallVector<mlir::acc::GPUParallelDimAttr>>
2752 parDimsPair = computeActiveAndInactiveParDims(
2753 op: interOp, block: &interOp.getRegion().front());
2754 if (hasFailed)
2755 return;
2756
2757 // If ThreadY reduction exists, subgroup alignment is applied
2758 // (blockDim.x = subgroupSize), so ThreadX lanes exist even without explicit
2759 // ThreadX parallelism. Add ThreadX to inactiveParDims if not already present.
2760 // Exception: if this region contains a thread-level (vector or worker)
2761 // routine call, all ThreadX threads must reach the call so the routine's
2762 // workgroup-wide barriers (e.g. shared memory alloca sync) converge.
2763 // A single-thread worker launch is left unaligned, so it has no such lanes.
2764 if (hasThreadYReduction && !isSingleThreadWorkerLaunch()) {
2765 MLIRContext *ctx = computeRegion->getContext();
2766 mlir::acc::GPUParallelDimAttr threadXParDim =
2767 mlir::acc::GPUParallelDimAttr::threadXDim(ctx);
2768 bool hasThreadXInActive =
2769 llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr pd) {
2770 return pd.isThreadX();
2771 });
2772 bool hasThreadXInInactive =
2773 llvm::any_of(Range&: parDimsPair.second, P: [](mlir::acc::GPUParallelDimAttr pd) {
2774 return pd.isThreadX();
2775 });
2776
2777 // Check if THIS predicate region contains a thread-level routine call.
2778 // We use the pre-computed hasThreadLevelRoutineCall as an early-out
2779 // optimization.
2780 bool regionHasThreadLevelRoutineCall = false;
2781 if (hasThreadLevelRoutineCall) {
2782 interOp.getRegion().walk(callback: [&](CallOpInterface callOp) {
2783 if (mlir::acc::GPUParallelDimAttr parDim =
2784 getAccRoutineCallParDim(callOp, policy: defaultPolicy)) {
2785 if (parDim.isThreadX() || parDim.isThreadY()) {
2786 regionHasThreadLevelRoutineCall = true;
2787 return WalkResult::interrupt();
2788 }
2789 }
2790 return WalkResult::advance();
2791 });
2792 }
2793
2794 if (!hasThreadXInActive && !hasThreadXInInactive &&
2795 !regionHasThreadLevelRoutineCall) {
2796 parDimsPair.second.push_back(Elt: threadXParDim);
2797 }
2798 }
2799
2800 Value predicate = emitPredicate(loc, inactiveParDims&: parDimsPair.second);
2801 // With one thread per block nothing is predicated, but a block-level
2802 // reduction store below must still become a cross-block atomic.
2803 if (!predicate && llvm::all_of(Range: computeRegion.getLaunchParDims(),
2804 P: [](mlir::acc::GPUParallelDimAttr pd) {
2805 return pd.isAnyBlock();
2806 }))
2807 predicate = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 1, width: 1);
2808 if (predicate) {
2809 LLVM_DEBUG(llvm::dbgs() << "predicate: " << predicate << "\n");
2810 bool isInsideThreadXLoop = false;
2811 bool isInsideThreadYLoop = false;
2812 for (auto parDim : parDimsPair.first) {
2813 if (parDim.isThreadX())
2814 isInsideThreadXLoop = true;
2815 if (parDim.isThreadY())
2816 isInsideThreadYLoop = true;
2817 }
2818 // Emits the reconvergence barrier matching this region's predication level,
2819 // at the current insertion point. Called for both before and after the
2820 // predicated store. Below is the pre-predicate barrier; the post-predicate
2821 // barrier is emitted after the ifOp.
2822 auto emitReconvergenceBarrier = [&]() {
2823 if (isInsideThreadXLoop) {
2824 // Inside ThreadX loop - skip barrier
2825 } else if (isInsideThreadYLoop) {
2826 // Inside ThreadY loop
2827 if (!isInsideACCSpecializedRoutine(op: computeRegion)) {
2828 // Add barrier if ThreadX is predicated (lane 0 writes must be
2829 // visible to all lanes before they read).
2830 bool predicatesThreadX = llvm::any_of(
2831 Range&: parDimsPair.second,
2832 P: [](mlir::acc::GPUParallelDimAttr pd) { return pd.isThreadX(); });
2833 if (predicatesThreadX)
2834 createPerRowBarrier(loc);
2835 }
2836 // For acc routine ThreadY routines, skip barrier
2837 } else if (!parDimsPair.first.empty()) {
2838 // Inside block loop
2839 createBarrier(loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get(
2840 context: interOp->getContext(), array: parDimsPair.first));
2841 } else {
2842 // Top level
2843 createBarrier(loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get(
2844 context: interOp->getContext(), array: parDimsPair.second));
2845 }
2846 };
2847
2848 // A gang-level (block, no thread) shared slot written here and reused on
2849 // the next iteration of an enclosing sequential loop must not be
2850 // overwritten before all threads read the current value. Emit the
2851 // reconvergence barrier before the store too (the post-store barrier below
2852 // only orders this iteration's store->read). Restricted to the block-level
2853 // case: such loops are run uniformly by every workgroup thread, so the
2854 // workgroup barrier cannot deadlock; worker/vector (thread_y/thread_x)
2855 // loops may have divergent trip counts.
2856 PrivateMemScope scope = needsPreStoreReuseBarrier(interOp);
2857 if (scope == PrivateMemScope::Gang)
2858 emitGPUBarrierWorkgroup(builder&: rewriter, loc);
2859 else if (scope == PrivateMemScope::Worker)
2860 createPerRowBarrier(loc);
2861
2862 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate,
2863 /*withElseRegion=*/false);
2864 Region &thenRegion = ifOp.getThenRegion();
2865 Block &thenBlock = thenRegion.back();
2866 rewriter.setInsertionPoint(thenBlock.getTerminator());
2867 // Ops in a predicate region may need to be further processed, recurse
2868 for (auto &bodyOp : interOp.getRegion().front().getOperations()) {
2869 // If the store's value loads from a block-level reduction
2870 // memref, convert to atomic for cross-block correctness.
2871 // Only at kernel top level (no active thread dims).
2872 if (memref::StoreOp storeOp = dyn_cast<memref::StoreOp>(Val: &bodyOp)) {
2873 std::optional<arith::AtomicRMWKind> blockReduceKind;
2874 bool failedReductionKind = false;
2875 Value storeVal = storeOp.getValueToStore();
2876 if (storeVal.getDefiningOp()) {
2877 // Walk the epilogue def-chain within the enclosing block to find
2878 // the block-level accumulate load feeding this store. Epilogue ops
2879 // (type conversions, arithmetic, etc.) are traversed transparently.
2880 // Non-acc loads and values defined outside the block are treated as
2881 // loop-invariant and return nullopt, bounding the search naturally.
2882 Block *epilogueBlock = interOp->getBlock();
2883 auto findBlockAccLoad =
2884 [&](auto &self,
2885 Value val) -> std::optional<arith::AtomicRMWKind> {
2886 Operation *def = val.getDefiningOp();
2887 if (!def || def->getBlock() != epilogueBlock)
2888 return std::nullopt;
2889 if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: def)) {
2890 for (auto *user : loadOp.getMemRef().getUsers()) {
2891 if (acc::ReductionAccumulateOp accOp =
2892 dyn_cast<acc::ReductionAccumulateOp>(Val: user)) {
2893 if (llvm::any_of(accOp.getParDims().getArray(),
2894 [](mlir::acc::GPUParallelDimAttr pd) {
2895 return pd.isAnyBlock();
2896 })) {
2897 FailureOr<arith::AtomicRMWKind> kind = getReductionKind(
2898 redOp: accOp.getReductionOperator(),
2899 type: accOp.getValue().getType(), loc: accOp.getLoc());
2900 if (failed(Result: kind)) {
2901 failedReductionKind = true;
2902 return std::nullopt;
2903 }
2904 return *kind;
2905 }
2906 }
2907 }
2908 return std::nullopt;
2909 }
2910 for (Value operand : def->getOperands())
2911 if (auto kind = self(self, operand))
2912 return kind;
2913 return std::nullopt;
2914 };
2915 blockReduceKind = findBlockAccLoad(findBlockAccLoad, storeVal);
2916 }
2917 if (failedReductionKind)
2918 return;
2919 // A private destination has a copy per block (or finer), so no other
2920 // block writes it and the store must stay a plain store.
2921 if (blockReduceKind && getPrivateScopeForMemref(memref: storeOp.getMemref()) !=
2922 PrivateMemScope::None)
2923 blockReduceKind.reset();
2924 if (blockReduceKind) {
2925 Value input = mapping.lookupOrDefault(from: storeOp.getValueToStore());
2926 Value memref = mapping.lookupOrDefault(from: storeOp.getMemref());
2927 bool threadIsActive = llvm::any_of(
2928 Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr pd) {
2929 return !pd.isAnyBlock();
2930 });
2931 if (!threadIsActive &&
2932 !isa_and_nonnull<memref::AllocaOp>(
2933 Val: unwrapMemRefConversion(v: memref).getDefiningOp())) {
2934 // Initialize the destination to the reduction identity before
2935 // cross-block atomics so that the final result reflects pure
2936 // assignment semantics (e.g. r = sum(a)), not accumulation
2937 // on top of the pre-kernel value.
2938 if (launch) {
2939 MemRefType memrefTy = cast<MemRefType>(Val: memref.getType());
2940 // Map the store indices for ranked memrefs.
2941 SmallVector<Value> initIndices;
2942 for (Value idx : storeOp.getIndices())
2943 initIndices.push_back(Elt: mapping.lookupOrDefault(from: idx));
2944
2945 OpBuilder::InsertionGuard guard(rewriter);
2946 Block &launchBody = launch.getBody().front();
2947 Operation *insertBefore = nullptr;
2948
2949 launchBody.walk(callback: [&](scf::ParallelOp parOp) -> WalkResult {
2950 for (Operation *parent = parOp->getParentOp(); parent;
2951 parent = parent->getParentOp()) {
2952 if (parent == launch.getOperation())
2953 break;
2954 if (isa<scf::ParallelOp>(Val: parent))
2955 return WalkResult::advance();
2956 }
2957 insertBefore = parOp.getOperation();
2958 return WalkResult::interrupt();
2959 });
2960 // The identity must be ordered before the atomics of every
2961 // block, so store it from a launch ahead of this one when the
2962 // address does not depend on the launch.
2963 std::function<bool(Value)> fromLaunch = [&](Value v) -> bool {
2964 if (auto arg = dyn_cast<BlockArgument>(Val&: v))
2965 return launch->isAncestor(other: arg.getOwner()->getParentOp());
2966 Operation *def = v.getDefiningOp();
2967 if (!launch->isAncestor(other: def) ||
2968 def->hasTrait<OpTrait::ConstantLike>())
2969 return false;
2970 // Ids and dims such as gpu.grid_dim take no operands.
2971 return def->getNumOperands() == 0 ||
2972 llvm::any_of(Range: def->getOperands(), P: fromLaunch);
2973 };
2974 if (!fromLaunch(memref) &&
2975 llvm::none_of(Range&: initIndices, P: fromLaunch)) {
2976 rewriter.setInsertionPoint(launch);
2977 Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
2978 Value token = launch.getAsyncToken();
2979 auto initLaunch = gpu::LaunchOp::create(
2980 builder&: rewriter, location: loc, gridSizeX: one, gridSizeY: one, gridSizeZ: one, blockSizeX: one, blockSizeY: one, blockSizeZ: one,
2981 /*dynamicSharedMemorySize=*/nullptr,
2982 asyncTokenType: token ? token.getType() : Type(),
2983 asyncDependencies: launch.getAsyncDependencies());
2984 rewriter.setInsertionPointToStart(
2985 &initLaunch.getBody().front());
2986 rewriter.setInsertionPoint(
2987 gpu::TerminatorOp::create(builder&: rewriter, location: loc));
2988 } else if (insertBefore)
2989 rewriter.setInsertionPoint(insertBefore);
2990 else
2991 rewriter.setInsertionPointToStart(&launchBody);
2992 // Recursively re-materialize operations whose definitions
2993 // do not dominate the insertion point. A single-level clone
2994 // is insufficient when the value is produced by a chain of
2995 // operations (e.g. reinterpret_cast depending on box_dims,
2996 // divsi, convert, etc.) that are all defined after the
2997 // insertion point.
2998 DominanceInfo domInfo(launch);
2999 IRMapping initMapping;
3000 std::function<Value(Value)> materialize =
3001 [&](Value val) -> Value {
3002 Operation *defOp = val.getDefiningOp();
3003 if (!defOp)
3004 return val;
3005 if (domInfo.dominates(a: defOp, b: &*rewriter.getInsertionPoint()))
3006 return val;
3007 if (auto mapped = initMapping.lookupOrNull(from: val))
3008 return mapped;
3009 // Recurse on operands; the recursive call seeds
3010 // initMapping for any operand it clones, which the
3011 // subsequent rewriter.clone(..., initMapping) picks up.
3012 for (Value operand : defOp->getOperands())
3013 materialize(operand);
3014 Operation *cloned = rewriter.clone(op&: *defOp, mapper&: initMapping);
3015 for (auto [orig, clonedRes] :
3016 llvm::zip(t: defOp->getResults(), u: cloned->getResults())) {
3017 initMapping.map(from: orig, to: clonedRes);
3018 }
3019 return initMapping.lookup(from: val);
3020 };
3021 Value initMemref = materialize(memref);
3022 for (auto &idx : initIndices)
3023 idx = materialize(idx);
3024 Value identityVal = createIdentityValue(
3025 b&: rewriter, loc, type: memrefTy.getElementType(), kind: *blockReduceKind,
3026 /*useOnlyFiniteValue=*/true);
3027 Value blockId = gpu::BlockIdOp::create(
3028 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
3029 Value threadId = gpu::ThreadIdOp::create(
3030 builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x);
3031 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
3032 Value isBlock0 = arith::CmpIOp::create(
3033 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: blockId, rhs: zero);
3034 Value isThread0 = arith::CmpIOp::create(
3035 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: threadId, rhs: zero);
3036 Value isFirstThread =
3037 arith::AndIOp::create(builder&: rewriter, location: loc, lhs: isBlock0, rhs: isThread0);
3038 auto initIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isFirstThread,
3039 /*withElseRegion=*/false);
3040 rewriter.setInsertionPoint(
3041 initIf.getThenRegion().back().getTerminator());
3042 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: identityVal, memref: initMemref,
3043 indices: initIndices);
3044 rewriter.setInsertionPointAfter(initIf);
3045 gpu::BarrierOp::create(builder&: rewriter, location: loc);
3046 }
3047 SmallVector<Value> atomicIndices;
3048 for (Value idx : storeOp.getIndices())
3049 atomicIndices.push_back(Elt: mapping.lookupOrDefault(from: idx));
3050 constructAtomicAccumulation(loc, memref, indices: atomicIndices, input,
3051 kind: *blockReduceKind);
3052 continue;
3053 }
3054 }
3055 }
3056 processOp(op: &bodyOp);
3057 }
3058 rewriter.setInsertionPointAfter(ifOp);
3059 emitReconvergenceBarrier();
3060 } else {
3061 // Ops in a predicate region may need to be further processed, recurse
3062 for (auto &bodyOp : interOp.getRegion().front().getOperations())
3063 processOp(op: &bodyOp);
3064 }
3065}
3066
3067// clang-format off
3068//
3069// Allocates private storage and broadcasts its pointer through a shared
3070// memref-of-memref slot, using the same predication utility as predicate
3071// regions.
3072//
3073// %0 = arith.cmpi eq, %thread_id_y, %c0 : index
3074// %1 = arith.cmpi eq, %thread_id_x, %c0 : index
3075// %2 = arith.andi %1, %0 : i1
3076// scf.if %2 {
3077// %alloc = memref.alloc() : memref<10xi32>
3078// memref.store %alloc, %arg1[] : memref<memref<10xi32>, #gpu.address_space<workgroup>>
3079// }
3080// gpu.barrier scope<subgroup>
3081// %4 = memref.load %arg1[] : memref<memref<10xi32>, #gpu.address_space<workgroup>>
3082//
3083// clang-format on
3084
3085Value ACCCGToGPULowering::allocatePrivatizeHeapStorage(
3086 acc::PrivatizeOp privatize, MemRefType baseTy,
3087 ValueRange mappedDynamicSizes) {
3088 Location loc = privatize.getLoc();
3089 Value mem =
3090 memref::AllocOp::create(builder&: rewriter, location: loc, memrefType: baseTy, dynamicSizes: mappedDynamicSizes);
3091 recordGlobalMemory(privatizeOp: privatize);
3092 OpBuilder::InsertionGuard guard(rewriter);
3093 Block *allocBlock = mem.getDefiningOp()->getBlock();
3094 if (allocBlock->mightHaveTerminator()) {
3095 rewriter.setInsertionPoint(allocBlock->getTerminator());
3096 memref::DeallocOp::create(builder&: rewriter, location: loc, memref: mem);
3097 }
3098 mapping.map(from: privatize.getResult(), to: mem);
3099 // Ops inside the kernel are rewritten from scratch. If privatize is
3100 // outside the kernel, it must be replaced with the allocated storage.
3101 if (!privatize->getParentOfType<acc::ComputeRegionOp>())
3102 rewriter.replaceOp(op: privatize, newValues: mem);
3103 return mem;
3104}
3105
3106Value ACCCGToGPULowering::processPrivatize(acc::PrivatizeOp privatize) {
3107 LLVM_DEBUG(llvm::dbgs() << "processing privatize: ";
3108 privatize->print(llvm::dbgs()); llvm::dbgs() << "\n");
3109 Value tracked = privatize.getResult();
3110 if (acc::ComputeRegionOp insUser =
3111 dyn_cast<acc::ComputeRegionOp>(Val: getOnlyUser(v: tracked))) {
3112 assert(privatize->hasOneUse() &&
3113 "expected acc.privatize op to have one use");
3114 tracked = insUser.getBody()->getArgument(
3115 i: privatize->use_begin()->getOperandNumber());
3116 }
3117 Operation *privatizeUser = getOnlyUser(v: tracked);
3118 assert(privatizeUser && "expected PrivateLocalOp user for privatize");
3119 acc::PrivateLocalOp privateLocalUser =
3120 dyn_cast<acc::PrivateLocalOp>(Val: privatizeUser);
3121
3122 std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>,
3123 SmallVector<mlir::acc::GPUParallelDimAttr>>
3124 parDimsPair = computeActiveAndInactiveParDims(op: privatizeUser, block: nullptr);
3125 // Set `par_dims` only when this `acc.privatize` does not already carry it.
3126 if (!privatize.getParDimsAttr()) {
3127 privatize.setParDimsAttr(mlir::acc::GPUParallelDimsAttr::get(
3128 context: rewriter.getContext(), array: parDimsPair.first));
3129 }
3130
3131 Location loc = privatize->getLoc();
3132 acc::PrivateType privTy = cast<acc::PrivateType>(Val: privatize.getType());
3133 ModuleOp module = computeRegion->getParentOfType<ModuleOp>();
3134 MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module);
3135
3136 gpu::GPUFuncOp gpuFuncOp = computeRegion->getParentOfType<gpu::GPUFuncOp>();
3137 // acc.privatize is outside this compute_region (e.g. passed via ins).
3138 // Leave the op unchanged here; processPrivateLocal materializes
3139 // storage when acc.private_local is lowered.
3140 if (!gpuFuncOp &&
3141 privatize->getParentOfType<acc::ComputeRegionOp>() != computeRegion) {
3142 return privatize.getResult();
3143 }
3144
3145 bool threadXActive =
3146 llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr parDim) {
3147 return parDim.isThreadX();
3148 });
3149 if (threadXActive &&
3150 canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) {
3151 auto alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy);
3152 recordThreadPrivate(privateLocal: privateLocalUser);
3153 mapping.map(from: privatize.getResult(), to: alloca.getResult());
3154 return alloca.getResult();
3155 }
3156
3157 if (!gpuFuncOp)
3158 return privatize.getResult();
3159
3160 // When ThreadY is active, shared memory must be indexed by ThreadY ID to
3161 // avoid races between ThreadY threads.
3162 // For ThreadX acc routines, assume ThreadY may be active since the routine
3163 // can be called from a ThreadY loop at runtime.
3164 // For ThreadY/block acc routines, do not force ThreadY indexing - variables
3165 // outside the ThreadY loop should be shared across ThreadY threads.
3166 bool threadYIsActive =
3167 llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr parDim) {
3168 return parDim.isThreadY();
3169 });
3170 // For routines with multiple ThreadY threads, need a workgroup barrier
3171 // instead of a subgroup barrier.
3172 // - ThreadX routines: called independently by different ThreadY threads, need
3173 // per-ThreadY slots (override threadYIsActive)
3174 // - ThreadY routines: workgroup barrier, but respect parDimsPair for
3175 // threadYIsActive
3176 // (variables before ThreadY loop are shared, inside are per-ThreadY)
3177 // - Block routines: single call, ThreadY threads cooperate within the
3178 // routine, so
3179 // variables at routine level are shared across ThreadY threads (single
3180 // slot)
3181 bool needsWorkgroupBarrier = false;
3182 if (isInsideACCSpecializedRoutine(op: computeRegion)) {
3183 FunctionOpInterface funcOp =
3184 computeRegion->getParentOfType<FunctionOpInterface>();
3185 mlir::acc::GPUParallelDimAttr routineParDim =
3186 getSpecializedRoutineDim(funcOp, policy: defaultPolicy);
3187 if (routineParDim.isThreadX()) {
3188 // ThreadX routine: always per-ThreadY slots since called from ThreadY
3189 // loops
3190 threadYIsActive = true;
3191 } else if (routineParDim.isThreadY()) {
3192 // ThreadY routine: workgroup barrier, but keep original threadYIsActive
3193 // (variables before ThreadY loop shared, inside per-ThreadY)
3194 needsWorkgroupBarrier = true;
3195 } else if (routineParDim.isAnyBlock()) {
3196 needsWorkgroupBarrier = true;
3197 }
3198 }
3199
3200 llvm::SmallVector<Value> mappedDynamicSizes;
3201 for (auto dynamicSize : privatize.getDynamicSizes()) {
3202 Value mappedDynamicSize = mapping.lookupOrDefault(from: dynamicSize);
3203 mappedDynamicSizes.push_back(Elt: mappedDynamicSize);
3204 }
3205 if (isInsideACCSpecializedRoutine(op: computeRegion) &&
3206 computeRegion.isEffectivelySerial()) {
3207 if (mappedDynamicSizes.empty()) {
3208 // Static sizes: use alloca (stack allocation)
3209 auto alloca =
3210 memref::AllocaOp::create(builder&: rewriter, location: privatize->getLoc(), memrefType: baseTy);
3211 recordThreadPrivate(privateLocal: privateLocalUser);
3212 mapping.map(from: privatize.getResult(), to: alloca.getResult());
3213 return alloca.getResult();
3214 }
3215 // Dynamic sizes: use alloc (heap allocation) with dealloc
3216 return allocatePrivatizeHeapStorage(privatize, baseTy, mappedDynamicSizes);
3217 }
3218
3219 if (threadXActive) {
3220 // Reached only after canUseStackAlloca failed: either the static size is
3221 // over the per-thread stack budget, or the shape is dynamic so the size
3222 // is unknown. Heap-allocate in both cases so a large runtime extent
3223 // cannot overflow the stack. Each lane still owns its own copy.
3224 return allocatePrivatizeHeapStorage(privatize, baseTy, mappedDynamicSizes);
3225 }
3226
3227 // Predication - when threadYIsActive, don't predicate on ThreadY dimension
3228 // since each ThreadY needs to execute the allocation for its own slot
3229 SmallVector<mlir::acc::GPUParallelDimAttr> predicateDims;
3230 for (auto parDim : parDimsPair.second) {
3231 // Skip ThreadY if threadYIsActive - each ThreadY needs to allocate
3232 if (threadYIsActive && parDim.isThreadY())
3233 continue;
3234 predicateDims.push_back(Elt: parDim);
3235 }
3236 Value predicate = emitPredicate(loc, inactiveParDims&: predicateDims);
3237 if (!predicate) {
3238 predicate = arith::ConstantOp::create(
3239 builder&: rewriter, location: loc, value: rewriter.getIntegerAttr(type: rewriter.getI1Type(), value: 1));
3240 }
3241 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate,
3242 /*withElseRegion=*/false);
3243 Region &thenRegion = ifOp.getThenRegion();
3244 Block &thenBlock = thenRegion.back();
3245 rewriter.setInsertionPoint(thenBlock.getTerminator());
3246 auto mem = memref::AllocOp::create(builder&: rewriter, location: privatize->getLoc(), memrefType: baseTy,
3247 dynamicSizes: mappedDynamicSizes);
3248 recordGlobalMemory(privatizeOp: privatize);
3249 // Shared memory allocation
3250 gpu::AddressSpaceAttr sharedMemoryAddressSpace = gpu::AddressSpaceAttr::get(
3251 context: computeRegion->getContext(), value: gpu::GPUDialect::getWorkgroupAddressSpace());
3252 // When ThreadY is active, create a shared memory array indexed by ThreadY ID.
3253 // Each ThreadY stores to its own slot; ThreadX lanes within a ThreadY share
3254 // it.
3255 constexpr int64_t kMaxThreadY = 32;
3256 MemRefType sharedMemTy =
3257 threadYIsActive
3258 ? MemRefType::get(shape: {kMaxThreadY}, elementType: baseTy, layout: MemRefLayoutAttrInterface{},
3259 memorySpace: sharedMemoryAddressSpace)
3260 : MemRefType::get(shape: {}, elementType: baseTy, layout: MemRefLayoutAttrInterface{},
3261 memorySpace: sharedMemoryAddressSpace);
3262 // The slot only transiently broadcasts the storage pointer, so reuse one per
3263 // type across privatizes, barriering (before the predicated store) on reuse.
3264 bool reuseBroadcast = !gpuFuncOp.isKernel();
3265 Value alloca;
3266 llvm::DenseMap<Type, Value>::iterator cachedSlot =
3267 reuseBroadcast ? privatizeBroadcastCache.find(Val: sharedMemTy)
3268 : privatizeBroadcastCache.end();
3269 if (reuseBroadcast && cachedSlot != privatizeBroadcastCache.end()) {
3270 alloca = cachedSlot->second;
3271 OpBuilder::InsertionGuard guard(rewriter);
3272 rewriter.setInsertionPoint(ifOp);
3273 mlir::acc::GPUParallelDimAttr dim =
3274 needsWorkgroupBarrier
3275 ? mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext())
3276 : mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext());
3277 createBarrier(
3278 loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get(context: rewriter.getContext(), array: {dim}));
3279 } else {
3280 alloca = gpuFuncOp.addWorkgroupAttribution(type: sharedMemTy,
3281 loc: rewriter.getUnknownLoc());
3282 // Setting the alignment to 16 because of a bug in the gpu toolchain.
3283 // The default alignment is 8, but optimizations create a packed store of 16
3284 // bytes which cause a misalignment error at runtime.
3285 unsigned index = gpuFuncOp.getNumWorkgroupAttributions() - 1;
3286 gpuFuncOp.setWorkgroupAttributionAttr(index,
3287 name: LLVM::LLVMDialect::getAlignAttrName(),
3288 value: rewriter.getI32IntegerAttr(value: 16));
3289 if (reuseBroadcast)
3290 privatizeBroadcastCache[sharedMemTy] = alloca;
3291 }
3292 // Store to shared memory, indexed by ThreadY ID when ThreadY is active.
3293 if (threadYIsActive) {
3294 Value threadYId = getThreadId(loc, dim: gpu::Dimension::y);
3295 memref::StoreOp::create(builder&: rewriter, location: privatize->getLoc(), valueToStore: mem, memref: alloca,
3296 indices: ValueRange{threadYId});
3297 } else {
3298 memref::StoreOp::create(builder&: rewriter, location: privatize->getLoc(), valueToStore: mem, memref: alloca);
3299 }
3300
3301 // Sync and load - use workgroup barrier for Block/ThreadY routines,
3302 // ThreadX barrier for ThreadX-only routines
3303 rewriter.setInsertionPointAfter(ifOp);
3304 if (needsWorkgroupBarrier) {
3305 mlir::acc::GPUParallelDimsAttr threadYDimsAttr =
3306 mlir::acc::GPUParallelDimsAttr::get(
3307 context: rewriter.getContext(),
3308 array: {mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext())});
3309 createBarrier(loc, parDimsAttr: threadYDimsAttr);
3310 } else {
3311 // ThreadX-only: use per-ThreadY barrier
3312 mlir::acc::GPUParallelDimsAttr threadXDimsAttr =
3313 mlir::acc::GPUParallelDimsAttr::get(
3314 context: rewriter.getContext(),
3315 array: {mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext())});
3316 createBarrier(loc, parDimsAttr: threadXDimsAttr);
3317 }
3318 // Load from shared memory, indexed by ThreadY ID when ThreadY is active.
3319 Value load;
3320 if (threadYIsActive) {
3321 Value threadYId = getThreadId(loc, dim: gpu::Dimension::y);
3322 load = memref::LoadOp::create(builder&: rewriter, location: privatize->getLoc(), memref: alloca,
3323 indices: ValueRange{threadYId});
3324 } else {
3325 load = memref::LoadOp::create(builder&: rewriter, location: privatize->getLoc(), memref: alloca,
3326 indices: ValueRange{});
3327 }
3328 rewriter.setInsertionPointAfter(load.getDefiningOp());
3329 mapping.map(from: privatize.getResult(), to: load);
3330
3331 // Operations inside the kernel are all rewritten from scratch.
3332 // But if the privatize op is outside the kernel, it needs to be replaced.
3333 if (!privatize->getParentOfType<acc::ComputeRegionOp>())
3334 rewriter.replaceOp(op: privatize, newValues: load);
3335 // Deallocate
3336 rewriter.setInsertionPoint(ifOp->getBlock()->getTerminator());
3337 if (needsWorkgroupBarrier) {
3338 mlir::acc::GPUParallelDimsAttr workerDimsAttr =
3339 mlir::acc::GPUParallelDimsAttr::get(
3340 context: rewriter.getContext(),
3341 array: {mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext())});
3342 createBarrier(loc, parDimsAttr: workerDimsAttr);
3343 } else {
3344 mlir::acc::GPUParallelDimsAttr vectorDimsAttr =
3345 mlir::acc::GPUParallelDimsAttr::get(
3346 context: rewriter.getContext(),
3347 array: {mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext())});
3348 createBarrier(loc, parDimsAttr: vectorDimsAttr);
3349 }
3350 auto ifOp2 = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate,
3351 /*withElseRegion=*/false);
3352 Region &thenRegion2 = ifOp2.getThenRegion();
3353 Block &thenBlock2 = thenRegion2.back();
3354 rewriter.setInsertionPoint(thenBlock2.getTerminator());
3355 memref::DeallocOp::create(builder&: rewriter, location: privatize->getLoc(), memref: load);
3356
3357 // Return the private memory
3358 rewriter.setInsertionPointAfter(load.getDefiningOp());
3359
3360 return load;
3361}
3362
3363// Materialize acc.private_local storage from acc.privatize: per-thread alloca
3364// when possible, otherwise a shared broadcast slot or acc.gpu_shared_memory.
3365
3366void ACCCGToGPULowering::processPrivateLocal(
3367 acc::PrivateLocalOp privateLocal, std::optional<int64_t> sharedMemCopies) {
3368 LLVM_DEBUG(llvm::dbgs() << "processing private local: ";
3369 privateLocal->print(llvm::dbgs()); llvm::dbgs() << "\n");
3370 Location loc = privateLocal.getLoc();
3371 acc::PrivateType privTy =
3372 cast<acc::PrivateType>(Val: privateLocal.getPrivatized().getType());
3373 ModuleOp module = computeRegion->getParentOfType<ModuleOp>();
3374 MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module);
3375 MemRefType byteMemrefTy =
3376 MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI8Type());
3377
3378 acc::PrivatizeOp privatizeOp = getPrivatizeOp(privateLocal, computeRegion);
3379 Value inputMem;
3380 if (privatizeOp->getParentOfType<acc::ComputeRegionOp>() == computeRegion) {
3381 inputMem = mapping.lookupOrNull(from: privatizeOp);
3382 if (inputMem) {
3383 Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem,
3384 resultType: privateLocal.getType());
3385 mapping.map(from: privateLocal.getResult(), to: result);
3386 return;
3387 }
3388 } else {
3389 // Hoisted acc.privatize: allocate per-thread stack storage in the launch
3390 // body. Cross-thread array reduction accumulators are per-thread too, so
3391 // the accumulate can reduce each element across threads. Skip when storage
3392 // par_dims lack thread_x (gang-/worker-scoped array temp).
3393 acc::ReductionAccumulateArrayOp arrayAccum =
3394 perThreadArrayReductionAccum(v: privateLocal.getResult());
3395 if ((isThreadXPrivatize(privatize: privatizeOp) ||
3396 (arrayAccum && storageHasThreadX(privateLocal, computeRegion))) &&
3397 canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) {
3398 Value alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy);
3399 recordThreadPrivate(privateLocal);
3400 if (arrayAccum) {
3401 FailureOr<arith::AtomicRMWKind> kind = getReductionKind(
3402 redOp: arrayAccum.getReductionOperator(), type: baseTy.getElementType(), loc);
3403 if (failed(Result: kind))
3404 return;
3405 initPerThreadArrayAccum(b&: rewriter, loc, alloca, baseTy, kind: *kind);
3406 }
3407 Value mem = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: alloca,
3408 resultType: privateLocal.getType());
3409 mapping.map(from: privateLocal.getResult(), to: mem);
3410 return;
3411 }
3412
3413 // If acc.privatize is outside the kernel, it needs to be converted
3414 // explicitly.
3415 std::optional<int64_t> copies =
3416 sharedMemCopies ? sharedMemCopies
3417 : isEligibleForSharedMemory(privateLocal, baseTy);
3418 if (copies) {
3419 int64_t numCopies = *copies;
3420 int64_t elementSize = getElementSizeInBytes(loc, elementType: baseTy.getElementType());
3421 int64_t numElements = 1;
3422 for (int64_t dim : baseTy.getShape())
3423 numElements *= dim;
3424 int64_t upperBound = elementSize * numElements * numCopies;
3425
3426 if (tryAllocateSharedMemory(bytes: upperBound)) {
3427 std::string varName =
3428 accSupport.getVariableName(v: privateLocal.getResult());
3429 sharedMemPrivateVarNames.push_back(Elt: varName.empty() ? "(*)" : varName);
3430
3431 gpu::AddressSpaceAttr workgroupAS = gpu::AddressSpaceAttr::get(
3432 context: computeRegion->getContext(),
3433 value: gpu::GPUDialect::getWorkgroupAddressSpace());
3434 MemRefType sharedMemTy =
3435 MemRefType::get(shape: baseTy.getShape(), elementType: baseTy.getElementType(),
3436 layout: MemRefLayoutAttrInterface{}, memorySpace: workgroupAS);
3437 Value sharedMem = acc::GPUSharedMemoryOp::create(
3438 builder&: rewriter, location: loc, resultType: sharedMemTy, numCopies: rewriter.getI64IntegerAttr(value: numCopies),
3439 staticUpperBoundBytes: rewriter.getI64IntegerAttr(value: upperBound), dynamicSizes: ValueRange{}, dynamicSharedMemoryScalingBytes: IntegerAttr{},
3440 dynamicSharedMemoryFixedBytes: IntegerAttr{});
3441
3442 Value mem =
3443 castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: sharedMem, resultType: baseTy);
3444 Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: mem,
3445 resultType: privateLocal.getType());
3446
3447 mapping.map(from: privateLocal.getResult(), to: result);
3448 return;
3449 }
3450 }
3451
3452 OpBuilder::InsertionGuard guard(rewriter);
3453 rewriter.setInsertionPoint(privatizeOp);
3454 inputMem = processPrivatize(privatize: privatizeOp);
3455 }
3456
3457 if (isInsideACCSpecializedRoutine(op: computeRegion)) {
3458 assert(inputMem && "expected input mem to be mapped");
3459 Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem,
3460 resultType: privateLocal.getType());
3461 mapping.map(from: privateLocal.getResult(), to: result);
3462 return;
3463 }
3464
3465 // The private element shape augmented with a dimension for each level of
3466 // parallelism
3467 SmallVector<int64_t> viewShape;
3468 // The dynamic sizes of the view (num_gangs, num_workers, vector_length)
3469 SmallVector<Value> viewDynSizes;
3470 // The offset of the subview from gpu.block_id/gpu.thread_id dimensions.
3471 SmallVector<OpFoldResult> subviewOffset;
3472 // The sizes of the subview where the dimensionality is brought back to the
3473 // private element. That is size 1 for each active block/thread dimension.
3474 SmallVector<OpFoldResult> subviewSizes;
3475 // The strides of the subview
3476 SmallVector<int64_t> subviewStrides;
3477 // The shape of the subview
3478 SmallVector<int64_t> subviewShape;
3479
3480 std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>,
3481 SmallVector<mlir::acc::GPUParallelDimAttr>>
3482 parDimsPair = computeActiveAndInactiveParDims(op: privateLocal, block: nullptr);
3483 acc::ReductionAccumulateArrayOp arrayAccum =
3484 perThreadArrayReductionAccum(v: privateLocal.getResult());
3485 for (mlir::acc::GPUParallelDimAttr parDim : parDimsPair.first) {
3486 if ((parDim.isThreadX() ||
3487 (arrayAccum && storageHasThreadX(privateLocal, computeRegion))) &&
3488 canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) {
3489 Value alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy);
3490 recordThreadPrivate(privateLocal);
3491 if (arrayAccum) {
3492 FailureOr<arith::AtomicRMWKind> kind = getReductionKind(
3493 redOp: arrayAccum.getReductionOperator(), type: baseTy.getElementType(), loc);
3494 if (failed(Result: kind))
3495 return;
3496 initPerThreadArrayAccum(b&: rewriter, loc, alloca, baseTy, kind: *kind);
3497 }
3498 Value mem = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: alloca,
3499 resultType: privateLocal.getType());
3500 mapping.map(from: privateLocal.getResult(), to: mem);
3501 return;
3502 }
3503 }
3504 if (parDimsPair.first.empty()) {
3505 // No parallelism is found above. It's single block execution.
3506 mlir::acc::insertParDim(
3507 parDims&: parDimsPair.first,
3508 parDim: mlir::acc::GPUParallelDimAttr::blockXDim(privateLocal.getContext()));
3509 }
3510 for (mlir::acc::GPUParallelDimAttr parDim : parDimsPair.first) {
3511 gpu::Processor gpuProc = parDim.getProcessor();
3512 Value gpuSize = getGPUSizeFor(proc: gpuProc);
3513 viewDynSizes.push_back(Elt: gpuSize);
3514 viewShape.push_back(Elt: ShapedType::kDynamic);
3515 subviewOffset.push_back(Elt: getGPUThreadIdFor(proc: gpuProc));
3516 subviewSizes.push_back(Elt: rewriter.getIndexAttr(value: 1));
3517 }
3518
3519 SmallVector<Value> innerDynSizes =
3520 resolvePrivateLocalDynamicExtents(privateLocal);
3521
3522 unsigned dynIdx = 0;
3523 for (auto innerDim : baseTy.getShape()) {
3524 subviewOffset.push_back(Elt: rewriter.getIndexAttr(value: 0));
3525 viewShape.push_back(Elt: innerDim);
3526 subviewShape.push_back(Elt: innerDim);
3527 if (innerDim == ShapedType::kDynamic) {
3528 assert(dynIdx < innerDynSizes.size() &&
3529 "not enough dynamic sizes for inner dimensions");
3530 viewDynSizes.push_back(Elt: innerDynSizes[dynIdx]);
3531 subviewSizes.push_back(Elt: innerDynSizes[dynIdx]);
3532 ++dynIdx;
3533 } else {
3534 subviewSizes.push_back(Elt: rewriter.getIndexAttr(value: innerDim));
3535 }
3536 }
3537
3538 // Do the strides in reverse order.
3539 int64_t stride = 1;
3540 for (auto innerDimIt = baseTy.getShape().rbegin();
3541 innerDimIt != baseTy.getShape().rend(); ++innerDimIt) {
3542 int64_t innerDim = *innerDimIt;
3543 subviewStrides.insert(I: subviewStrides.begin(), Elt: stride);
3544 if (innerDim == ShapedType::kDynamic)
3545 stride = ShapedType::kDynamic;
3546 if (stride != ShapedType::kDynamic)
3547 stride *= innerDim;
3548 }
3549
3550 Value memBuffer =
3551 castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem, resultType: byteMemrefTy);
3552 auto c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
3553 MemRefType viewType = MemRefType::get(shape: viewShape, elementType: baseTy.getElementType());
3554 auto view = memref::ViewOp::create(builder&: rewriter, location: loc, resultType0: viewType, source: memBuffer,
3555 byte_shift: c0.getResult(), sizes: viewDynSizes);
3556
3557 // memref.subview
3558 StridedLayoutAttr stridedLayout = StridedLayoutAttr::get(
3559 context: computeRegion->getContext(), offset: ShapedType::kDynamic, strides: subviewStrides);
3560 MemRefType subviewType =
3561 MemRefType::get(shape: subviewShape, elementType: baseTy.getElementType(), layout: stridedLayout);
3562 SmallVector<OpFoldResult> ones(viewType.getRank(), rewriter.getIndexAttr(value: 1));
3563 Value subview = memref::SubViewOp::create(builder&: rewriter, location: loc, resultType: subviewType, source: view,
3564 offsets: subviewOffset, sizes: subviewSizes, strides: ones);
3565
3566 // Pointer-like casts cannot carry memref offsets. Shift the aligned pointer
3567 // so later zero-offset views retain this thread's private slice.
3568 auto metadata =
3569 memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: subview);
3570 Value elementBytes = arith::ConstantIndexOp::create(
3571 builder&: rewriter, location: loc, value: getElementSizeInBytes(loc, elementType: baseTy.getElementType()));
3572 Value byteOffset =
3573 arith::MulIOp::create(builder&: rewriter, location: loc, lhs: metadata.getOffset(), rhs: elementBytes);
3574 Value privateView = memref::ViewOp::create(builder&: rewriter, location: loc, resultType0: baseTy, source: memBuffer,
3575 byte_shift: byteOffset, sizes: innerDynSizes);
3576 Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: privateView,
3577 resultType: privateLocal.getType());
3578 mapping.map(from: privateLocal.getResult(), to: result);
3579}
3580
3581// Could be scf::for or scf::parallel
3582template <typename LoopOp>
3583void ACCCGToGPULowering::processSeqLoop(LoopOp loopOp) {
3584 // Pre-process shared-memory-eligible private_local ops. Only direct-child
3585 // ops are considered; nested private_local ops (e.g. inside predicate_region)
3586 // are handled by recursive body processing.
3587 LLVM_DEBUG(llvm::dbgs() << "processing seq loop: ";
3588 loopOp->print(llvm::dbgs()); llvm::dbgs() << "\n");
3589 llvm::SmallPtrSet<Operation *, 4> preProcessedPrivateLocals;
3590 ModuleOp module = computeRegion->getParentOfType<ModuleOp>();
3591 for (auto &bodyOp : loopOp.getBody()->getOperations()) {
3592 if (acc::PrivateLocalOp privateLocal =
3593 dyn_cast<acc::PrivateLocalOp>(&bodyOp)) {
3594 acc::PrivateType privTy =
3595 cast<acc::PrivateType>(Val: privateLocal.getPrivatized().getType());
3596 MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module);
3597 if (auto copies = isEligibleForSharedMemory(privateLocal, baseTy)) {
3598 processPrivateLocal(privateLocal, sharedMemCopies: copies);
3599 preProcessedPrivateLocals.insert(Ptr: privateLocal.getOperation());
3600 }
3601 }
3602 }
3603
3604 LoopOp newLoop =
3605 dyn_cast<LoopOp>(rewriter.cloneWithoutRegions(*loopOp, mapping));
3606 rewriter.createBlock(
3607 &newLoop.getRegion(), newLoop.getRegion().begin(),
3608 loopOp.getBody()->getArgumentTypes(),
3609 SmallVector<Location>(loopOp.getBody()->getArgumentTypes().size(),
3610 loopOp->getLoc()));
3611 rewriter.setInsertionPointToStart(&newLoop.getRegion().front());
3612
3613 // Need to clone all block arguments
3614 Block::BlockArgListType blockArgs = loopOp.getBody()->getArguments();
3615 assert(blockArgs.size() && "expected block arguments for loop");
3616 mapping.map(blockArgs, newLoop.getBody()->getArguments());
3617
3618 // A gang-private slot written here is one copy per workgroup, reused by every
3619 // iteration, and a thread that has already started the next iteration would
3620 // overwrite the value another thread has not read yet. Separate the
3621 // iterations with a barrier at the top of the body.
3622 if (needsInLoopReuseBarrier(loopOp: loopOp.getOperation()) &&
3623 !isInsideConditional(block: rewriter.getInsertionBlock()))
3624 emitGPUBarrierWorkgroup(rewriter, loopOp->getLoc());
3625
3626 for (auto &bodyOp : loopOp.getBody()->getOperations()) {
3627 if (preProcessedPrivateLocals.contains(Ptr: &bodyOp))
3628 continue;
3629 processOp(op: &bodyOp);
3630 }
3631
3632 mapping.map(loopOp.getResults(), newLoop.getResults());
3633 rewriter.setInsertionPointAfter(newLoop);
3634
3635 // Postpone the barrier when trailing work in this block still separates the
3636 // loop from the next reconvergence point.
3637 if (hasTrailingSideEffectSiblings(loopOp.getOperation()))
3638 deferredBarrierSeqLoops.push_back(Elt: loopOp.getOperation());
3639 else
3640 createBarrierAfterSeqLoop(loopOp: loopOp.getOperation());
3641}
3642
3643void ACCCGToGPULowering::flushDeferredBarriersBefore(Operation *beforeOp) {
3644 Block *block = beforeOp->getBlock();
3645 SmallVector<Operation *, 4> toFlush;
3646 for (Operation *loopOp : deferredBarrierSeqLoops)
3647 if (loopOp->getBlock() == block && loopOp->isBeforeInBlock(other: beforeOp))
3648 toFlush.push_back(Elt: loopOp);
3649 if (toFlush.empty())
3650 return;
3651 llvm::sort(C&: toFlush,
3652 Comp: [](Operation *a, Operation *b) { return a->isBeforeInBlock(other: b); });
3653 for (Operation *loopOp : toFlush)
3654 createBarrierAfterSeqLoop(loopOp);
3655 deferredBarrierSeqLoops.erase(
3656 CS: std::remove_if(first: deferredBarrierSeqLoops.begin(),
3657 last: deferredBarrierSeqLoops.end(),
3658 pred: [&](Operation *loopOp) {
3659 return loopOp->getBlock() == block &&
3660 loopOp->isBeforeInBlock(other: beforeOp);
3661 }),
3662 CE: deferredBarrierSeqLoops.end());
3663}
3664
3665// try to process op as a loop mapped to a gpu parallelism
3666// failure signifies not a loop and needs different processing
3667void ACCCGToGPULowering::processParallelOp(scf::ParallelOp parallelOp) {
3668 LLVM_DEBUG(llvm::dbgs() << "processing par loop: ";
3669 parallelOp->print(llvm::dbgs()); llvm::dbgs() << "\n");
3670 assert(mlir::acc::hasParDimsAttr(parallelOp) &&
3671 "requires parallel dimensions attribute");
3672 mlir::acc::GPUParallelDimsAttr pDimsAttr =
3673 mlir::acc::getParDimsAttr(op: parallelOp);
3674 // both of these should be dealt with before compiler reaches ACCCGToGPU
3675 // Invalid parallel-loop structure should be rejected before acc-cg-to-gpu.
3676 assert(pDimsAttr.getArray().size() == 1 &&
3677 "expected a single par dim in acc-cg-to-gpu");
3678 assert(parallelOp.getInductionVars().size() == 1 &&
3679 "expected a single induction variable in acc-cg-to-gpu");
3680
3681 mlir::acc::GPUParallelDimAttr parDim = pDimsAttr.getArray().front();
3682
3683 bool savedGridStrideFlag = insideAccumulateGridStride;
3684 Value savedReductionBuf = reductionSharedBuf;
3685 if (parDim.isThreadX()) {
3686 bool found = false;
3687 parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) {
3688 bool hasBlockDim = false;
3689 bool hasThreadDim = false;
3690 for (auto d : accOp.getParDims().getArray()) {
3691 if (d.isAnyBlock())
3692 hasBlockDim = true;
3693 if (d.isThreadX() || d.isThreadY())
3694 hasThreadDim = true;
3695 }
3696 if (hasThreadDim && !hasBlockDim) {
3697 found = true;
3698 return WalkResult::interrupt();
3699 }
3700 return WalkResult::advance();
3701 });
3702 if (found)
3703 insideAccumulateGridStride = true;
3704 }
3705
3706 // common loop body processing for both par0 and 1+
3707 auto processLoopBody = [&]() {
3708 // process inner ops recursively
3709 for (auto &bodyOp : parallelOp.getBody()->getOperations()) {
3710 if (bodyOp.hasTrait<OpTrait::IsTerminator>()) {
3711 // Non-cloned parallel loops reconverge at their terminator.
3712 flushDeferredBarriersBefore(beforeOp: &bodyOp);
3713 continue;
3714 }
3715 // Non-terminator body ops are processed recursively.
3716 processOp(op: &bodyOp);
3717 }
3718 };
3719
3720 if (parDim.isSeq()) {
3721 LLVM_DEBUG(llvm::dbgs() << "loop: parDim: " << parDim << " as gpu seq\n");
3722 // Sequential loops here are remainder loops from partitioned parallel
3723 // loops - clone them as-is but process the body as a parallel region.
3724 // When blockDim.x >= subgroupSize and the loop contains a thread-level
3725 // accumulate, inactive grid-stride threads cannot participate in subgroup
3726 // reductions, so we use atomic-to-shared-memory reduction instead.
3727 bool needsAtomicReduction = false;
3728 bool hasAccumulateSibling = false;
3729 if (scf::ParallelOp parentPar =
3730 parallelOp->getParentOfType<scf::ParallelOp>()) {
3731 if (mlir::acc::GPUParallelDimsAttr parentDims =
3732 mlir::acc::getParDimsAttr(op: parentPar);
3733 parentDims && llvm::any_of(Range: parentDims.getArray(),
3734 P: [](auto d) { return d.isThreadX(); })) {
3735 for (auto &op : parentPar.getBody()->getOperations()) {
3736 if (acc::ReductionAccumulateOp acc =
3737 dyn_cast<acc::ReductionAccumulateOp>(Val&: op)) {
3738 bool hasBlockDim = false;
3739 bool hasThreadDim = false;
3740 for (auto d : acc.getParDims().getArray()) {
3741 if (d.isAnyBlock())
3742 hasBlockDim = true;
3743 if (d.isThreadX() || d.isThreadY())
3744 hasThreadDim = true;
3745 }
3746 if (hasThreadDim && !hasBlockDim)
3747 hasAccumulateSibling = true;
3748 }
3749 }
3750 }
3751 }
3752 if (insideAccumulateGridStride || hasAccumulateSibling) {
3753 for (auto launchArg : computeRegion.getLaunchArgs()) {
3754 if (acc::ParWidthOp pw = launchArg.getDefiningOp<acc::ParWidthOp>()) {
3755 if (pw.getParDim().isThreadX()) {
3756 if (auto cval = getConstantIntValue(ofr: pw.getLaunchArg()))
3757 needsAtomicReduction = (*cval >= options.subgroupSize);
3758 else
3759 needsAtomicReduction = true;
3760 break;
3761 }
3762 }
3763 }
3764 }
3765 if (needsAtomicReduction && !reductionSharedBuf) {
3766 Type elemTy;
3767 parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) {
3768 Type t = accOp.getValue().getType();
3769 if (isa<FloatType, IntegerType>(Val: t))
3770 elemTy = t;
3771 return elemTy ? WalkResult::interrupt() : WalkResult::advance();
3772 });
3773 if (!elemTy)
3774 needsAtomicReduction = false;
3775 }
3776 if (needsAtomicReduction && !reductionSharedBuf) {
3777 Location seqLoc = parallelOp->getLoc();
3778 gpu::AddressSpaceAttr workgroupAS = gpu::AddressSpaceAttr::get(
3779 context: computeRegion->getContext(),
3780 value: gpu::GPUDialect::getWorkgroupAddressSpace());
3781 Type elemTy;
3782 parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) {
3783 Type t = accOp.getValue().getType();
3784 if (isa<FloatType, IntegerType>(Val: t))
3785 elemTy = t;
3786 return elemTy ? WalkResult::interrupt() : WalkResult::advance();
3787 });
3788 assert(elemTy && "expected scalar reduction element type");
3789 unsigned elemBytes = elemTy.getIntOrFloatBitWidth() / 8;
3790 MemRefType bufTy = MemRefType::get(shape: {options.subgroupSize}, elementType: elemTy,
3791 map: AffineMap{}, memorySpace: workgroupAS);
3792 reductionSharedBuf = acc::GPUSharedMemoryOp::create(
3793 builder&: rewriter, location: seqLoc, resultType: bufTy, numCopies: rewriter.getI64IntegerAttr(value: 1),
3794 staticUpperBoundBytes: rewriter.getI64IntegerAttr(value: options.subgroupSize * elemBytes),
3795 dynamicSizes: ValueRange{}, dynamicSharedMemoryScalingBytes: IntegerAttr{}, dynamicSharedMemoryFixedBytes: IntegerAttr{});
3796 Value tidY = getThreadId(loc: seqLoc, dim: gpu::Dimension::y);
3797 Value identity;
3798 if (isa<FloatType>(Val: elemTy)) {
3799 identity = arith::ConstantOp::create(
3800 builder&: rewriter, location: seqLoc, result: elemTy, value: rewriter.getFloatAttr(type: elemTy, value: 0.0));
3801 } else {
3802 identity = arith::ConstantIntOp::create(builder&: rewriter, location: seqLoc, type: elemTy, value: 0);
3803 }
3804 memref::StoreOp::create(builder&: rewriter, location: seqLoc, valueToStore: identity, memref: reductionSharedBuf,
3805 indices: tidY);
3806 createPerRowBarrier(loc: seqLoc);
3807 }
3808 processSeqLoop(loopOp: parallelOp);
3809 loopReductions.push_back(Elt: parallelOp);
3810 } else {
3811 LLVM_DEBUG(llvm::dbgs()
3812 << "processing loop: parDim: " << parDim << " as gpu par\n");
3813 // actual parallel loops get their iv mapped to gpu hierarchy
3814 // and the loop construct is not cloned to gpu kernel only the
3815 // ops are cloned with mapping of gpu id for original loop iv
3816 Value gpuThreadId = getGPUThreadIdFor(proc: parDim.getProcessor());
3817 mapping.map(from: parallelOp.getInductionVars()[0], to: gpuThreadId);
3818
3819 processLoopBody();
3820
3821 // All threads of the row (or block) must finish their iterations before
3822 // any of them continues past the loop. The sequential nest around the loop
3823 // is executed by every thread, so without a barrier its later code can read
3824 // results that another thread has not stored yet.
3825 if (needsBarrierAfterThreadLoop(loop: parallelOp))
3826 createBarrier(loc: parallelOp->getLoc(), parDimsAttr: pDimsAttr);
3827
3828 // Since the loop is not copied over, create dummy mappings for_each
3829 // of the loop results. These will ultimately by replaced with a
3830 // reduction
3831 llvm::for_each(Range: parallelOp.getResults(), F: [&](Value v) {
3832 Type valTy = v.getType();
3833 TypedAttr zeroAttr = rewriter.getZeroAttr(type: valTy);
3834 auto zero = arith::ConstantOp::create(builder&: rewriter, location: parallelOp->getLoc(),
3835 result: valTy, value: zeroAttr);
3836 mapping.map(from: v, to: zero);
3837 });
3838 loopReductions.push_back(Elt: parallelOp);
3839 }
3840 insideAccumulateGridStride = savedGridStrideFlag;
3841 if (!insideAccumulateGridStride && !savedReductionBuf)
3842 reductionSharedBuf = Value();
3843}
3844
3845/// Map an atomic RMW kind to the corresponding `gpu.all_reduce` operation.
3846static gpu::AllReduceOperation
3847getAllReduceOperation(arith::AtomicRMWKind kind) {
3848 switch (kind) {
3849 case arith::AtomicRMWKind::addf:
3850 case arith::AtomicRMWKind::addi:
3851 return gpu::AllReduceOperation::ADD;
3852 case arith::AtomicRMWKind::mulf:
3853 case arith::AtomicRMWKind::muli:
3854 return gpu::AllReduceOperation::MUL;
3855 case arith::AtomicRMWKind::minu:
3856 return gpu::AllReduceOperation::MINUI;
3857 case arith::AtomicRMWKind::mins:
3858 return gpu::AllReduceOperation::MINSI;
3859 case arith::AtomicRMWKind::minnumf:
3860 return gpu::AllReduceOperation::MINNUMF;
3861 case arith::AtomicRMWKind::maxu:
3862 return gpu::AllReduceOperation::MAXUI;
3863 case arith::AtomicRMWKind::maxs:
3864 return gpu::AllReduceOperation::MAXSI;
3865 case arith::AtomicRMWKind::maxnumf:
3866 return gpu::AllReduceOperation::MAXNUMF;
3867 case arith::AtomicRMWKind::ori:
3868 return gpu::AllReduceOperation::OR;
3869 case arith::AtomicRMWKind::andi:
3870 return gpu::AllReduceOperation::AND;
3871 case arith::AtomicRMWKind::xori:
3872 return gpu::AllReduceOperation::XOR;
3873 case arith::AtomicRMWKind::minimumf:
3874 return gpu::AllReduceOperation::MINIMUMF;
3875 case arith::AtomicRMWKind::maximumf:
3876 return gpu::AllReduceOperation::MAXIMUMF;
3877 case arith::AtomicRMWKind::assign:
3878 break;
3879 }
3880 llvm_unreachable("unsupported atomic kind");
3881}
3882
3883void ACCCGToGPULowering::constructAtomicAccumulation(
3884 Location loc, Value memref, ValueRange indices, Value input,
3885 arith::AtomicRMWKind kind) {
3886 assert(!memref.getDefiningOp<memref::AllocaOp>() &&
3887 "cannot lower atomic accumulation on an stack variable");
3888
3889 // acc.atomic.update derives the element address from the memref descriptor's
3890 // base pointer and offset field; it has no subscript operand. When the store
3891 // being lowered targets a specific array element (e.g. result(idx) =
3892 // max(...)), fold the indices into the descriptor offset with a subview so
3893 // the atomic updates the intended element. Otherwise the atomic always hits
3894 // element 0, so any reduction whose destination index is non-zero is
3895 // miscompiled (the result lands in element 0 while the intended element keeps
3896 // its identity-init value).
3897 Value target = memref;
3898 if (!indices.empty()) {
3899 MemRefType memrefTy = cast<MemRefType>(Val: memref.getType());
3900 unsigned rank = memrefTy.getRank();
3901 assert(indices.size() == rank && "expected one index per memref dimension");
3902 SmallVector<OpFoldResult> offsets(indices.begin(), indices.end());
3903 SmallVector<OpFoldResult> sizes(rank, rewriter.getIndexAttr(value: 1));
3904 SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(value: 1));
3905 target = memref::SubViewOp::create(builder&: rewriter, location: loc, source: memref, offsets, sizes,
3906 strides);
3907 }
3908
3909 auto atomicUpdateOp =
3910 acc::AtomicUpdateOp::create(builder&: rewriter, location: loc, x: target, /*ifCond=*/Value());
3911 Region &region = atomicUpdateOp->getRegion(index: 0);
3912 Block *block =
3913 rewriter.createBlock(parent: &region, insertPt: region.begin(), argTypes: {input.getType()}, locs: {loc});
3914 rewriter.setInsertionPointToStart(block);
3915 Value reductionExpr =
3916 generateReductionOp(b&: rewriter, loc, lhs: input, rhs: block->getArgument(i: 0), kind);
3917 acc::YieldOp::create(builder&: rewriter, location: loc, operands: reductionExpr);
3918 rewriter.setInsertionPointAfter(atomicUpdateOp);
3919}
3920
3921void ACCCGToGPULowering::createGPUAllReduceOp(
3922 Location loc, Value input, Value memref, arith::AtomicRMWKind kind,
3923 mlir::acc::GPUParallelDimsAttr parDimsAttr, ValueRange indices,
3924 bool isPerThreadPrivateTarget) {
3925 gpu::AllReduceOperationAttr attr = gpu::AllReduceOperationAttr::get(
3926 context: computeRegion->getContext(), value: getAllReduceOperation(kind));
3927 auto emitAllReduce = [&](Value operand) -> Value {
3928 auto allReduceOp =
3929 gpu::AllReduceOp::create(builder&: rewriter, location: loc, value: operand, op: attr, uniform: true);
3930 mlir::acc::setParDimsAttr(op: allReduceOp, attr: parDimsAttr);
3931 return allReduceOp;
3932 };
3933
3934 Value reducedValue;
3935 if (auto complexType = dyn_cast<ComplexType>(Val: input.getType())) {
3936 // gpu.all_reduce takes only integers and floats, so a complex value has to
3937 // go through as components. Addition is the one operator reaching here
3938 // where that is equivalent: multiplication mixes the components, and
3939 // min/max have no ordering on complex at all.
3940 if (kind != arith::AtomicRMWKind::addf) {
3941 std::string msg;
3942 llvm::raw_string_ostream os(msg);
3943 os << "reduction: " << arith::stringifyAtomicRMWKind(kind) << " of "
3944 << complexType << " across threads";
3945 (void)accSupport.emitNYI(loc, message: os.str());
3946 hasFailed = true;
3947 return;
3948 }
3949 Type elementType = complexType.getElementType();
3950 Value realResult =
3951 emitAllReduce(complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: input));
3952 Value imagResult =
3953 emitAllReduce(complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: input));
3954 reducedValue = complex::CreateOp::create(builder&: rewriter, location: loc, complex: complexType,
3955 real: realResult, imaginary: imagResult);
3956 } else {
3957 reducedValue = emitAllReduce(input);
3958 }
3959 // Predicate the store on the thread-level dimensions being reduced so that
3960 // only one thread per reduced group writes the result. Only dimensions in
3961 // parDimsAttr are included; sweeping over all dimensions between the highest
3962 // par_dim and thread_x would incorrectly add unrelated dimensions (e.g.
3963 // thread_y for a thread_x-only reduction), preventing other rows from
3964 // storing their independent results.
3965 SmallVector<mlir::acc::GPUParallelDimAttr> inactiveParDims;
3966 MLIRContext *ctx = computeRegion->getContext();
3967 bool hasThreadX = false;
3968 for (auto parDim : parDimsAttr.getArray()) {
3969 if (parDim.isAnyBlock())
3970 continue;
3971 if (parDim.isThreadX())
3972 hasThreadX = true;
3973 if (computeRegion.getLaunchArg(parDim) ||
3974 isInsideACCSpecializedRoutine(op: computeRegion)) {
3975 inactiveParDims.push_back(Elt: parDim);
3976 }
3977 }
3978 // Subgroup alignment may introduce extra ThreadX lanes even when ThreadX is
3979 // not part of the reduction. Predicate on ThreadX so only one lane stores.
3980 // A single-thread worker launch is left unaligned, so it has no such lanes.
3981 if (!hasThreadX && !isSingleThreadWorkerLaunch())
3982 inactiveParDims.push_back(Elt: mlir::acc::GPUParallelDimAttr::threadXDim(ctx));
3983 Value predicate = emitPredicate(loc, inactiveParDims);
3984 // Predication is only needed when the store target is visible to
3985 // multiple threads (shared/global memory). Per-thread targets like
3986 // memref.alloca are thread-private: gpu.all_reduce returns the same
3987 // value on all threads, so each can safely store to its own copy.
3988 // Detect per-thread storage by walking through conversion ops to
3989 // find the underlying allocation.
3990 bool isPerThreadPrivate = isPerThreadPrivateTarget ||
3991 isa_and_nonnull<memref::AllocaOp>(
3992 Val: unwrapMemRefConversion(v: memref).getDefiningOp());
3993 // combine
3994 scf::IfOp ifOp;
3995 if (predicate && !isPerThreadPrivate) {
3996 ifOp =
3997 scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, /*withElseRegion=*/false);
3998 Region &thenRegion = ifOp.getThenRegion();
3999 Block &thenBlock = thenRegion.back();
4000 rewriter.setInsertionPoint(thenBlock.getTerminator());
4001 }
4002 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: reducedValue, memref, indices);
4003 if (predicate && !isPerThreadPrivate)
4004 rewriter.setInsertionPointAfter(ifOp);
4005 // A later block combine reuses this instead of reloading the slot. This only
4006 // applies to scalar accumulators; array elements are indexed individually.
4007 if (indices.empty())
4008 reductionAccumValue[memref] = reducedValue;
4009}
4010
4011void ACCCGToGPULowering::postprocessAccumulateOp(
4012 acc::ReductionAccumulateOp op) {
4013 Location loc = op->getLoc();
4014
4015 rewriter.setInsertionPoint(op);
4016
4017 // Check whether this accumulate has only block-level par dims (no thread
4018 // dims). gpu.all_reduce reduces across threads within a block, which is
4019 // wrong for block-only reductions - the loop result is already per-thread
4020 // and only needs a predicated store to the memref.
4021 bool hasThreadDim = false;
4022 SmallVector<mlir::acc::GPUParallelDimAttr> threadParDims;
4023 for (auto parDim : op.getParDims().getArray()) {
4024 if (!parDim.isAnyBlock()) {
4025 hasThreadDim = true;
4026 threadParDims.push_back(Elt: parDim);
4027 }
4028 }
4029
4030 std::optional<arith::AtomicRMWKind> kind;
4031 if (hasThreadDim) {
4032 FailureOr<arith::AtomicRMWKind> kindOr = getReductionKind(
4033 redOp: op.getReductionOperator(), type: op.getValue().getType(), loc);
4034 if (failed(Result: kindOr))
4035 return;
4036 kind = *kindOr;
4037 }
4038
4039 if (hasThreadDim && reductionSharedBuf &&
4040 op.getValue().getType() ==
4041 cast<MemRefType>(Val: reductionSharedBuf.getType()).getElementType()) {
4042 Value val = op.getValue();
4043 Value mem = op.getMemref();
4044 Value tidY = getThreadId(loc, dim: gpu::Dimension::y);
4045 memref::AtomicRMWOp::create(builder&: rewriter, location: loc, kind: *kind, value: val, memref: reductionSharedBuf,
4046 indices: ValueRange{tidY});
4047 createPerRowBarrier(loc);
4048 Value result =
4049 memref::LoadOp::create(builder&: rewriter, location: loc, memref: reductionSharedBuf, indices: tidY);
4050 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: result, memref: mem);
4051 reductionAccumValue[mem] = result;
4052 } else if (hasThreadDim) {
4053 createGPUAllReduceOp(loc, input: op.getValue(), memref: op.getMemref(), kind: *kind,
4054 parDimsAttr: op.getParDims());
4055 // Nothing replaced the accumulate, so leave it in place rather than
4056 // erasing it below.
4057 if (hasFailed)
4058 return;
4059 } else {
4060 // Block-only: no gpu.all_reduce needed (all threads have the same
4061 // value after broadcast). Just store the value to the memref.
4062 // Predicate on all thread dims when the target is shared memory.
4063 Value val = mapping.lookupOrDefault(from: op.getValue());
4064 Value mem = mapping.lookupOrDefault(from: op.getMemref());
4065 bool isPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>(
4066 Val: unwrapMemRefConversion(v: mem).getDefiningOp());
4067 if (!isPerThreadPrivate) {
4068 SmallVector<mlir::acc::GPUParallelDimAttr> predDims;
4069 for (auto parDim : computeRegion.getLaunchParDims())
4070 if (!parDim.isAnyBlock())
4071 predDims.push_back(Elt: parDim);
4072 if (predDims.empty()) {
4073 predDims.push_back(Elt: mlir::acc::GPUParallelDimAttr::threadXDim(
4074 computeRegion->getContext()));
4075 }
4076 Value predicate = emitPredicate(loc, inactiveParDims&: predDims);
4077 auto ifOp =
4078 scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, /*withElseRegion=*/false);
4079 rewriter.setInsertionPoint(ifOp.getThenRegion().back().getTerminator());
4080 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: val, memref: mem);
4081 rewriter.setInsertionPointAfter(ifOp);
4082 } else {
4083 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: val, memref: mem);
4084 }
4085 }
4086
4087 // erase acc.reduction_accumulate
4088 rewriter.eraseOp(op);
4089}
4090
4091void ACCCGToGPULowering::postprocessLoopReduction(scf::ParallelOp parLoop) {
4092 if (parLoop.getNumReductions() == 0)
4093 return;
4094
4095 for (unsigned i = 0; i < parLoop.getNumResults(); ++i) {
4096 for (Operation *user :
4097 mapping.lookupOrDefault(from: parLoop.getResult(i)).getUsers()) {
4098 if (acc::ReductionAccumulateOp accumulateOp =
4099 dyn_cast<acc::ReductionAccumulateOp>(Val: user)) {
4100 postprocessAccumulateOp(op: accumulateOp);
4101 }
4102 }
4103 }
4104}
4105
4106void ACCCGToGPULowering::processExecuteRegion(scf::ExecuteRegionOp op) {
4107 LLVM_DEBUG(llvm::dbgs() << "processing execute region op: ";
4108 op->print(llvm::dbgs()); llvm::dbgs() << "\n");
4109 Location loc = op->getLoc();
4110 auto types = op.getResultTypes();
4111 Region &oldRegion = op.getRegion();
4112 // create the executeRegion op inside gpu launch
4113 auto executeRegionOp = scf::ExecuteRegionOp::create(builder&: rewriter, location: loc, resultType0: types);
4114 Region &region = executeRegionOp.getRegion();
4115 rewriter.createBlock(parent: &region);
4116 rewriter.setInsertionPointToEnd(&region.front());
4117
4118 llvm::DenseMap<Block *, Block *> blockMap;
4119 blockMap[&oldRegion.front()] = &region.front();
4120
4121 // Create blocks in the new operation corresponding to all blocks in the
4122 // original op
4123 for (auto &oldBlock : llvm::drop_begin(RangeOrContainer&: oldRegion.getBlocks())) {
4124 TypeRange argTypes = oldBlock.getArgumentTypes();
4125 size_t numArgs = argTypes.size();
4126 // Create new block with same argument types
4127 Block *newBlock = rewriter.createBlock(parent: &region, insertPt: region.end(), argTypes,
4128 locs: SmallVector<Location>(numArgs, loc));
4129 blockMap[&oldBlock] = newBlock;
4130 // Map block arguments
4131 mapping.map(from: oldBlock.getArguments(), to: newBlock->getArguments());
4132 }
4133
4134 // Iterate over all blocks of oldRegion and all operations inside them
4135 // process all the ops except the terminator
4136 for (auto [oldBlock, newBlock] :
4137 llvm::zip(t&: oldRegion.getBlocks(), u&: region.getBlocks())) {
4138 OpBuilder::InsertionGuard blockGuard(rewriter);
4139 rewriter.setInsertionPointToStart(&newBlock);
4140 for (auto &bodyOp : oldBlock.getOperations()) {
4141 // Skip terminators during normal iteration - handle them separately
4142 if (bodyOp.hasTrait<OpTrait::IsTerminator>())
4143 continue;
4144 processOp(op: &bodyOp);
4145 }
4146
4147 // Copy the terminator from old block to new block
4148 Operation *oldTerminator = oldBlock.getTerminator();
4149 rewriter.setInsertionPointToEnd(&newBlock);
4150 Operation *newTerminator = rewriter.clone(op&: *oldTerminator, mapper&: mapping);
4151
4152 // Replace successors with mapped blocks
4153 for (unsigned i = 0; i < oldTerminator->getNumSuccessors(); ++i) {
4154 Block *oldDest = oldTerminator->getSuccessor(index: i);
4155 Block *newDest = blockMap.lookup(Val: oldDest);
4156 assert(newDest && "Successor block must be in blockMap");
4157 newTerminator->setSuccessor(block: newDest, index: i);
4158 }
4159 }
4160 mapping.map(from: op->getResults(), to: executeRegionOp->getResults());
4161 rewriter.setInsertionPointAfter(executeRegionOp);
4162}
4163
4164void ACCCGToGPULowering::processAccumulateOp(acc::ReductionAccumulateOp op) {
4165 LLVM_DEBUG(llvm::dbgs() << "processing accumulate op: " << *op << "\n");
4166 Value accumulateValue = op.getValue();
4167 if (reductionSharedBuf &&
4168 mapping.lookupOrDefault(from: accumulateValue).getType() ==
4169 cast<MemRefType>(Val: reductionSharedBuf.getType()).getElementType()) {
4170 Location loc = op->getLoc();
4171 FailureOr<arith::AtomicRMWKind> kind = getReductionKind(
4172 redOp: op.getReductionOperator(), type: accumulateValue.getType(), loc);
4173 if (failed(Result: kind))
4174 return;
4175 Value mappedValue = mapping.lookupOrDefault(from: accumulateValue);
4176 Value memref = mapping.lookupOrDefault(from: op.getMemref());
4177 Value tidY = getThreadId(loc, dim: gpu::Dimension::y);
4178 memref::AtomicRMWOp::create(builder&: rewriter, location: loc, kind: *kind, value: mappedValue,
4179 memref: reductionSharedBuf, indices: ValueRange{tidY});
4180 createPerRowBarrier(loc);
4181 Value result =
4182 memref::LoadOp::create(builder&: rewriter, location: loc, memref: reductionSharedBuf, indices: tidY);
4183 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: result, memref);
4184 reductionAccumValue[memref] = result;
4185 return;
4186 }
4187 if (accumulateValue.getDefiningOp<scf::ParallelOp>()) {
4188 Operation *newOp = rewriter.clone(op&: *op, mapper&: mapping);
4189 mapping.map(from: op->getResults(), to: newOp->getResults());
4190 } else if (isRedundantChainAccumulate(op)) {
4191 // The destination memref already holds the correctly aggregated value
4192 // (atomically reduced by a preceding acc.reduction_combine with a block
4193 // par_dim). Skip the redundant gpu.all_reduce + atomic.update; emit a
4194 // Workgroup-wide barrier so downstream readers see all prior atomic
4195 // updates.
4196 LLVM_DEBUG(llvm::dbgs() << " skipped: redundant chain accumulate\n");
4197 gpu::BarrierOp::create(builder&: rewriter, location: op->getLoc());
4198 } else {
4199 Value mappedValue = mapping.lookupOrDefault(from: accumulateValue);
4200 Value memref = mapping.lookupOrDefault(from: op.getMemref());
4201 FailureOr<arith::AtomicRMWKind> kind = getReductionKind(
4202 redOp: op.getReductionOperator(), type: accumulateValue.getType(), loc: op.getLoc());
4203 if (failed(Result: kind))
4204 return;
4205 createGPUAllReduceOp(loc: op->getLoc(), input: mappedValue, memref, kind: *kind,
4206 parDimsAttr: op.getParDims());
4207 }
4208}
4209
4210/// True when \p v takes a distinct value per thread: it derives from a thread
4211/// id, either directly or through the bounds of an enclosing loop. By this
4212/// point a thread-mapped loop carries its mapping in its bounds rather than in
4213/// `acc.par_dims`, so the bounds are what must be inspected.
4214static bool isThreadVarying(Value v, ArrayRef<Value> threadIds,
4215 DenseSet<Value> &visited) {
4216 if (!v || !visited.insert(V: v).second)
4217 return false;
4218 if (llvm::is_contained(Range&: threadIds, Element: v))
4219 return true;
4220 if (auto arg = dyn_cast<BlockArgument>(Val&: v)) {
4221 Operation *owner = arg.getOwner()->getParentOp();
4222 unsigned dim = arg.getArgNumber();
4223 if (auto loop = dyn_cast<scf::ParallelOp>(Val: owner)) {
4224 if (dim >= loop.getLowerBound().size())
4225 return false;
4226 return isThreadVarying(v: loop.getLowerBound()[dim], threadIds, visited) ||
4227 isThreadVarying(v: loop.getStep()[dim], threadIds, visited);
4228 }
4229 if (auto loop = dyn_cast<scf::ForOp>(Val: owner))
4230 return dim == 0 &&
4231 (isThreadVarying(v: loop.getLowerBound(), threadIds, visited) ||
4232 isThreadVarying(v: loop.getStep(), threadIds, visited));
4233 return false;
4234 }
4235 Operation *def = v.getDefiningOp();
4236 if (!def)
4237 return false;
4238 if (isa<gpu::ThreadIdOp, gpu::LaneIdOp>(Val: def))
4239 return true;
4240 return llvm::any_of(Range: def->getOperands(), P: [&](Value o) {
4241 return isThreadVarying(v: o, threadIds, visited);
4242 });
4243}
4244
4245/// Strips view and memory-space-cast chains to the underlying buffer.
4246static Value accumulatorRoot(Value v) {
4247 while (Operation *op = v.getDefiningOp()) {
4248 if (auto cast = dyn_cast<memref::MemorySpaceCastOp>(Val: op)) {
4249 v = cast.getSource();
4250 continue;
4251 }
4252 if (auto viewLike = dyn_cast<ViewLikeOpInterface>(Val: op)) {
4253 if (isa<MemRefType>(Val: viewLike.getViewSource().getType())) {
4254 v = viewLike.getViewSource();
4255 continue;
4256 }
4257 }
4258 break;
4259 }
4260 return v;
4261}
4262
4263/// Matches `%l = load %m[%i]` / `%c = combine(%l, %x)` / `store %c, %m[%i]` on
4264/// the accumulator \p accum and returns the contributed value `%x`.
4265static Value matchAccumulatorUpdate(memref::StoreOp store, Value accum) {
4266 if (accumulatorRoot(v: store.getMemRef()) != accum)
4267 return {};
4268 Operation *combine = store.getValueToStore().getDefiningOp();
4269 if (!combine || combine->getNumOperands() != 2)
4270 return {};
4271 for (unsigned i = 0; i != 2; ++i) {
4272 auto load = combine->getOperand(idx: i).getDefiningOp<memref::LoadOp>();
4273 if (!load || accumulatorRoot(v: load.getMemRef()) != accum)
4274 continue;
4275 if (!llvm::equal(LRange: load.getIndices(), RRange: store.getIndices()))
4276 continue;
4277 return combine->getOperand(idx: 1 - i);
4278 }
4279 return {};
4280}
4281
4282/// A block-shared accumulator is updated in place by the loop body, so several
4283/// threads may hit the same element. Make those updates atomic unless the
4284/// element index provably varies across the participating threads.
4285static void atomicizeSharedAccumulatorUpdates(Value accum,
4286 arith::AtomicRMWKind kind,
4287 ArrayRef<Value> threadIds,
4288 RewriterBase &rewriter) {
4289 OpBuilder::InsertionGuard guard(rewriter);
4290 SmallVector<memref::StoreOp> stores;
4291 SmallVector<Value> worklist{accum};
4292 DenseSet<Value> seen;
4293 while (!worklist.empty()) {
4294 Value cur = worklist.pop_back_val();
4295 if (!seen.insert(V: cur).second)
4296 continue;
4297 for (Operation *user : cur.getUsers()) {
4298 if (auto store = dyn_cast<memref::StoreOp>(Val: user))
4299 stores.push_back(Elt: store);
4300 else if (isa<ViewLikeOpInterface, memref::MemorySpaceCastOp>(Val: user))
4301 llvm::append_range(C&: worklist, R: user->getResults());
4302 }
4303 }
4304
4305 for (memref::StoreOp store : stores) {
4306 Value contribution = matchAccumulatorUpdate(store, accum);
4307 if (!contribution)
4308 continue;
4309 // A thread-varying index means each thread owns its element, so the
4310 // existing plain update is already race-free.
4311 if (llvm::any_of(Range: store.getIndices(), P: [&](Value idx) {
4312 DenseSet<Value> visited;
4313 return isThreadVarying(v: idx, threadIds, visited);
4314 }))
4315 continue;
4316 Operation *combine = store.getValueToStore().getDefiningOp();
4317 rewriter.setInsertionPoint(store);
4318 memref::AtomicRMWOp::create(builder&: rewriter, location: store.getLoc(), kind, value: contribution,
4319 memref: store.getMemRef(), indices: store.getIndices());
4320 rewriter.eraseOp(op: store);
4321 if (combine && combine->use_empty())
4322 rewriter.eraseOp(op: combine);
4323 }
4324}
4325
4326void ACCCGToGPULowering::processAccumulateArrayOp(
4327 acc::ReductionAccumulateArrayOp op) {
4328 LLVM_DEBUG(llvm::dbgs() << "processing accumulate array op: " << *op << "\n");
4329 Location loc = op.getLoc();
4330
4331 Value memref = mapping.lookupOrDefault(from: op.getMemref());
4332 MemRefType memrefTy = dyn_cast<MemRefType>(Val: memref.getType());
4333 if (!memrefTy) {
4334 (void)accSupport.emitNYI(loc, message: "reduction: non-MemRefTy accumulate array");
4335 return;
4336 }
4337 FailureOr<arith::AtomicRMWKind> kindOr = getReductionKind(
4338 redOp: op.getReductionOperator(), type: memrefTy.getElementType(), loc);
4339 if (failed(Result: kindOr))
4340 return;
4341 arith::AtomicRMWKind kind = *kindOr;
4342
4343 // The (already mapped/cloned) acc.bounds op describes the element range; it
4344 // is dead after lowering since we read its operands directly.
4345 acc::DataBoundsOp boundsOp = mapping.lookupOrDefault(from: op.getBounds())
4346 .getDefiningOp<acc::DataBoundsOp>();
4347 assert(boundsOp && "expected acc.bounds defining op for array accumulate");
4348 auto eraseDeadBounds = [&] {
4349 if (boundsOp->use_empty())
4350 rewriter.eraseOp(op: boundsOp);
4351 };
4352
4353 auto isUnitExtent = [&](mlir::acc::GPUParallelDimAttr parDim) {
4354 std::optional<uint64_t> width =
4355 computeRegion.getKnownConstantLaunchArg(parDim);
4356 return width && *width == 1;
4357 };
4358
4359 // A dimension launched one wide partitions nothing, so skip it.
4360 bool hasThreadDim = false;
4361 for (auto pd : op.getParDims().getArray()) {
4362 if (isUnitExtent(pd))
4363 continue;
4364 hasThreadDim |= pd.isAnyThread();
4365 }
4366
4367 // Without a thread dimension no two threads share an accumulator copy, so
4368 // each copy already holds a complete total and nothing has to be merged
4369 // here. Merging across gangs, if any, is left to the combine.
4370 if (!hasThreadDim) {
4371 eraseDeadBounds();
4372 return;
4373 }
4374
4375 // Per-element gpu.all_reduce is only correct when each thread owns its own
4376 // accumulator copy. For a statically-shaped accumulator, classify from the
4377 // operand: an explicit shared/heap allocation is block-shared regardless of
4378 // size, and a stack alloca (or a view over one) is per-thread when it fits
4379 // the per-thread stack budget. Storage `acc.par_dims` without thread_x means
4380 // gang-/worker-scoped privacy (shared among vector lanes) even if the type
4381 // would fit on the stack. For a dynamically-shaped accumulator the type
4382 // conveys no size, so classify from storage/accumulate thread_x dims.
4383 auto storageIsThreadXPrivate = [&](Value v) -> bool {
4384 acc::PrivateLocalOp privateLocal = getPrivateLocalForMemref(memref: v);
4385 GPUParallelDimsAttr dims =
4386 privateLocal ? getPrivateParDims(privateLocal, computeRegion)
4387 : GPUParallelDimsAttr();
4388 if (!dims) {
4389 if (Operation *root = unwrapMemRefConversion(v).getDefiningOp())
4390 dims = getParDimsAttr(op: root);
4391 }
4392 return !dims ||
4393 llvm::any_of(Range: dims.getArray(), P: [](auto d) { return d.isThreadX(); });
4394 };
4395 Operation *rootOp = unwrapMemRefConversion(v: memref).getDefiningOp();
4396 bool isSharedStorage = isa_and_nonnull<memref::AllocOp>(Val: rootOp) ||
4397 isa_and_nonnull<acc::GPUSharedMemoryOp>(Val: rootOp);
4398 if (auto addrSpace = dyn_cast_if_present<gpu::AddressSpaceAttr>(
4399 Val: memrefTy.getMemorySpace())) {
4400 isSharedStorage |=
4401 addrSpace.getValue() == gpu::GPUDialect::getWorkgroupAddressSpace();
4402 }
4403 bool isPerThreadPrivate =
4404 !isSharedStorage && storageIsThreadXPrivate(op.getMemref()) &&
4405 (memrefTy.hasStaticShape()
4406 ? canUseStackAlloca(baseTy: memrefTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)
4407 : llvm::any_of(Range: op.getParDims().getArray(),
4408 P: [](mlir::acc::GPUParallelDimAttr d) {
4409 return d.isThreadX();
4410 }));
4411 if (!isPerThreadPrivate) {
4412 // Block-shared accumulator: the body already updated it in place, so the
4413 // block partial is complete and the atomic combine finishes across blocks.
4414 // Threads that share an element must not race, so their in-place updates
4415 // become atomic.
4416 SmallVector<Value> threadIds;
4417 if (Value xId = getGPUThreadIdFor(proc: gpu::Processor::ThreadX))
4418 threadIds.push_back(Elt: xId);
4419 if (Value yId = getGPUThreadIdFor(proc: gpu::Processor::ThreadY))
4420 threadIds.push_back(Elt: yId);
4421 if (Value zId = getGPUThreadIdFor(proc: gpu::Processor::ThreadZ))
4422 threadIds.push_back(Elt: zId);
4423 atomicizeSharedAccumulatorUpdates(accum: accumulatorRoot(v: memref), kind, threadIds,
4424 rewriter);
4425 eraseDeadBounds();
4426 return;
4427 }
4428
4429 // Bounds are normalized to be zero-based.
4430 auto toIndex = [&](Value v) -> Value {
4431 if (v.getType().isIndex())
4432 return v;
4433 return arith::IndexCastOp::create(builder&: rewriter, location: loc, out: rewriter.getIndexType(),
4434 in: v);
4435 };
4436
4437 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
4438 Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
4439 Value lb =
4440 boundsOp.getLowerbound() ? toIndex(boundsOp.getLowerbound()) : zero;
4441 Value step = boundsOp.getStride() ? toIndex(boundsOp.getStride()) : one;
4442 // Exclusive upper bound. `extent` counts elements, so the span is
4443 // `extent * step` (for the common unit-stride case step is 1); fall back to
4444 // the inclusive upperbound when no extent is given.
4445 Value ub;
4446 if (boundsOp.getExtent()) {
4447 Value span = arith::MulIOp::create(builder&: rewriter, location: loc,
4448 lhs: toIndex(boundsOp.getExtent()), rhs: step);
4449 ub = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: lb, rhs: span);
4450 } else {
4451 assert(boundsOp.getUpperbound() &&
4452 "acc.bounds must specify an extent or upperbound");
4453 ub = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: toIndex(boundsOp.getUpperbound()),
4454 rhs: one);
4455 }
4456
4457 // Reduce each array element across the requested parallel dimensions.
4458 auto forOp = scf::ForOp::create(builder&: rewriter, location: loc, lowerBound: lb, upperBound: ub, step);
4459 {
4460 OpBuilder::InsertionGuard guard(rewriter);
4461 rewriter.setInsertionPoint(forOp.getBody()->getTerminator());
4462 Value iv = forOp.getInductionVar();
4463 SmallVector<Value> indices{iv};
4464 if (memrefTy.getRank() > 1) {
4465 indices.resize(N: memrefTy.getRank());
4466 Value linearIndex = iv;
4467 for (int64_t dim = memrefTy.getRank() - 1; dim >= 0; --dim) {
4468 Value dimSize =
4469 memrefTy.isDynamicDim(idx: dim)
4470 ? memref::DimOp::create(builder&: rewriter, location: loc, source: memref, index: dim).getResult()
4471 : arith::ConstantIndexOp::create(builder&: rewriter, location: loc,
4472 value: memrefTy.getDimSize(idx: dim))
4473 .getResult();
4474 indices[dim] =
4475 arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: linearIndex, rhs: dimSize);
4476 if (dim != 0)
4477 linearIndex =
4478 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: linearIndex, rhs: dimSize);
4479 }
4480 }
4481 Value elem = memref::LoadOp::create(builder&: rewriter, location: loc, memref, indices);
4482 createGPUAllReduceOp(loc, input: elem, memref, kind, parDimsAttr: op.getParDims(), indices,
4483 /*isPerThreadPrivateTarget=*/true);
4484 }
4485
4486 eraseDeadBounds();
4487}
4488
4489void ACCCGToGPULowering::processReductionOp(acc::ReductionInitOp op) {
4490 // Clone the inner ops of the reduction op only
4491 op.getRegion().walk<WalkOrder::PreOrder>(callback: [&](Operation *innerOp) {
4492 if (acc::YieldOp yieldOp = dyn_cast<acc::YieldOp>(Val: innerOp)) {
4493 op.getResult().replaceAllUsesWith(newValue: mapping.lookup(from: yieldOp.getOperand(i: 0)));
4494 return WalkResult::interrupt();
4495 }
4496 if (innerOp->getNumRegions() > 0) {
4497 processOp(op: innerOp);
4498 return WalkResult::skip();
4499 }
4500 rewriter.clone(op&: *innerOp, mapper&: mapping);
4501 return WalkResult::advance();
4502 });
4503}
4504
4505void ACCCGToGPULowering::processReductionCombineOp(acc::ReductionCombineOp op) {
4506 LLVM_DEBUG(llvm::dbgs() << "processing reduction combine op: ";
4507 op->print(llvm::dbgs()); llvm::dbgs() << "\n");
4508 Location loc = op.getLoc();
4509 MemRefType memrefType = dyn_cast<MemRefType>(Val: op.getSrcMemref().getType());
4510 assert(memrefType && "expected memref type for reduction combine op");
4511 assert(memrefType.getRank() == 0 &&
4512 "expected scalar memref type for reduction combine op");
4513 Type elTy = memrefType.getElementType();
4514 FailureOr<arith::AtomicRMWKind> kindOr =
4515 getReductionKind(redOp: op.getReductionOperator(), type: elTy, loc);
4516 if (failed(Result: kindOr))
4517 return;
4518 arith::AtomicRMWKind kind = *kindOr;
4519
4520 Value srcMemref = mapping.lookupOrDefault(from: op.getSrcMemref());
4521 Value destMemref = mapping.lookupOrDefault(from: op.getDestMemref());
4522
4523 // A block par_dim normally means the accumulator is shared across blocks and
4524 // must be updated atomically. But when the destination resolves to a
4525 // thread-private stack alloca (e.g. an inner-loop reduction combining one
4526 // private accumulator into another private accumulator for the same thread),
4527 // the update is not visible to other threads and must not be atomic. Treat
4528 // such destinations as a plain load/combine/store below.
4529 bool destIsPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>(
4530 Val: unwrapMemRefConversion(v: destMemref).getDefiningOp());
4531
4532 SmallVector<mlir::acc::GPUParallelDimAttr> parDims =
4533 getReductionCombineParDims(op);
4534 // An atomic is needed exactly when more than one block can reach the
4535 // destination. A block dimension in the reduction's parallel dims implies
4536 // that; so does a thread-only reduction in a multi-block launch, where every
4537 // block combines its own partial into the same shared slot.
4538 bool destIsGridShared =
4539 !destIsPerThreadPrivate &&
4540 (llvm::any_of(Range&: parDims,
4541 P: [](mlir::acc::GPUParallelDimAttr parDim) {
4542 return parDim.isAnyBlock();
4543 }) ||
4544 computeRegion.launchesMultipleBlocks());
4545 if (destIsGridShared) {
4546 // The predication (tid.x == 0 when subgroup-aligned) is already handled
4547 // by the parent predicate_region processing.
4548 // Reloading a grid-shared slot races with other blocks; record it and
4549 // replace with the block-reduced register value in the fixup.
4550 auto srcLoad =
4551 memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{});
4552 pendingCombineReloads.push_back(Elt: {srcMemref, srcLoad});
4553 constructAtomicAccumulation(loc, memref: destMemref, /*indices=*/{}, input: srcLoad, kind);
4554 return;
4555 }
4556
4557 // Atomic construction is not needed; lower this operation to typical
4558 // reduction update operations. E.g. dest = dest <kind> src
4559 auto srcLoad = memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{});
4560 auto destLoad =
4561 memref::LoadOp::create(builder&: rewriter, location: loc, memref: destMemref, indices: ValueRange{});
4562 Value combine = generateReductionOp(b&: rewriter, loc, lhs: srcLoad, rhs: destLoad, kind);
4563 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: combine, memref: destMemref, indices: ValueRange{});
4564}
4565
4566void ACCCGToGPULowering::processCombineRegionOp(
4567 acc::ReductionCombineRegionOp op) {
4568 LLVM_DEBUG(llvm::dbgs() << "processing combine region op: ";
4569 op->print(llvm::dbgs()); llvm::dbgs() << "\n");
4570 // A block par_dim on a combine into a thread-private stack alloca is not a
4571 // real cross-block accumulation (the alloca is not shared across blocks), so
4572 // it must use a plain load/combine/store rather than an atomic update.
4573 bool destIsPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>(
4574 Val: unwrapMemRefConversion(v: mapping.lookupOrDefault(from: op.getDestVar()))
4575 .getDefiningOp());
4576 SmallVector<mlir::acc::GPUParallelDimAttr> parDims =
4577 getReductionCombineParDims(op);
4578 for (auto parDim : parDims) {
4579 if (parDim.isAnyBlock() && !destIsPerThreadPrivate) {
4580 // Block reduction directly stores to the accumulator using atomic.
4581 // The predication (tid.x == 0 when subgroup-aligned) is already handled
4582 // by the parent predicate_region processing.
4583 for (Operation *user : op.getSrcVar().getUsers()) {
4584 if (acc::ReductionAccumulateOp accumulateOp =
4585 dyn_cast<acc::ReductionAccumulateOp>(Val: user)) {
4586 Location loc = accumulateOp.getLoc();
4587 FailureOr<arith::AtomicRMWKind> kind =
4588 getReductionKind(redOp: accumulateOp.getReductionOperator(),
4589 type: accumulateOp.getValue().getType(), loc);
4590 if (failed(Result: kind))
4591 return;
4592 Value srcMemref = mapping.lookupOrDefault(from: accumulateOp.getMemref());
4593 // Recorded and patched in the fixup to avoid the reload race.
4594 auto reductionLoad =
4595 memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{});
4596 pendingCombineReloads.push_back(Elt: {srcMemref, reductionLoad});
4597 constructAtomicAccumulation(loc,
4598 memref: mapping.lookupOrDefault(from: op.getDestVar()),
4599 /*indices=*/{}, input: reductionLoad, kind: *kind);
4600 return;
4601 }
4602 }
4603 // For decomposed complex reductions, the AccumulateOp was replaced
4604 // with real/imag AccumulateOps. Load from the private memref which
4605 // holds the reconstructed complex value.
4606 Value privateMemref = mapping.lookupOrDefault(from: op.getSrcVar());
4607 MemRefType memrefTy = cast<MemRefType>(Val: privateMemref.getType());
4608 if (isa<ComplexType>(Val: memrefTy.getElementType())) {
4609 Location loc = op.getLoc();
4610 Value reductionResult =
4611 memref::LoadOp::create(builder&: rewriter, location: loc, memref: privateMemref, indices: ValueRange{});
4612 arith::AtomicRMWKind kind = arith::AtomicRMWKind::addf;
4613 op.getRegion().walk(callback: [&](Operation *innerOp) {
4614 if (isa<complex::MulOp>(Val: innerOp))
4615 kind = arith::AtomicRMWKind::mulf;
4616 });
4617 constructAtomicAccumulation(loc,
4618 memref: mapping.lookupOrDefault(from: op.getDestVar()),
4619 /*indices=*/{}, input: reductionResult, kind);
4620 return;
4621 }
4622 }
4623 }
4624 op.getRegion().walk<WalkOrder::PreOrder>(callback: [&](Operation *innerOp) {
4625 if (acc::YieldOp yieldOp = dyn_cast<acc::YieldOp>(Val: innerOp))
4626 return WalkResult::interrupt();
4627 if (innerOp->getNumRegions() > 0) {
4628 processOp(op: innerOp);
4629 return WalkResult::skip();
4630 }
4631 rewriter.clone(op&: *innerOp, mapper&: mapping);
4632 return WalkResult::advance();
4633 });
4634}
4635
4636void ACCCGToGPULowering::processGenericOp(Operation *op) {
4637 // Operations with no regions or operations for which we know
4638 // no recursive processing is needed can be fully cloned.
4639 LLVM_DEBUG(llvm::dbgs() << "processing generic op, cloning: ";
4640 op->print(llvm::dbgs()); llvm::dbgs() << "\n");
4641 Operation *newOp = rewriter.clone(op&: *op, mapper&: mapping);
4642 // update mapping as cloning creates different result values
4643 mapping.map(from: op->getResults(), to: newOp->getResults());
4644}
4645
4646void ACCCGToGPULowering::processGenericOpWithRegions(Operation *op) {
4647 // Generic handling for operations with regions
4648 LLVM_DEBUG(llvm::dbgs() << "processing generic op with regions: ";
4649 op->print(llvm::dbgs()); llvm::dbgs() << "\n");
4650
4651 // Clone the operation structure without its regions
4652 Operation *newOp = rewriter.cloneWithoutRegions(op&: *op, mapper&: mapping);
4653
4654 // Process each region recursively
4655 for (auto [oldRegion, newRegion] :
4656 llvm::zip(t: op->getRegions(), u: newOp->getRegions())) {
4657 // Create blocks in the new region corresponding to old region blocks
4658 for (auto &oldBlock : oldRegion.getBlocks()) {
4659 TypeRange argTypes = oldBlock.getArgumentTypes();
4660 size_t numArgs = argTypes.size();
4661 // Create new block with same argument types
4662 Block *newBlock =
4663 rewriter.createBlock(parent: &newRegion, insertPt: newRegion.end(), argTypes,
4664 locs: SmallVector<Location>(numArgs, op->getLoc()));
4665
4666 // Map block arguments
4667 mapping.map(from: oldBlock.getArguments(), to: newBlock->getArguments());
4668
4669 // Process each operation in the block
4670 for (auto &innerOp : oldBlock.getOperations()) {
4671 OpBuilder::InsertionGuard guard(rewriter);
4672 rewriter.setInsertionPointToEnd(newBlock);
4673 processOp(op: &innerOp);
4674 }
4675 }
4676 }
4677 rewriter.setInsertionPointAfter(newOp);
4678
4679 // Update mapping for results
4680 mapping.map(from: op->getResults(), to: newOp->getResults());
4681}
4682
4683// thread through par dim to verify redundant/0 execution modes
4684void ACCCGToGPULowering::processOp(Operation *op) {
4685 if (isDeferredBarrierFlushPoint(op))
4686 flushDeferredBarriersBefore(beforeOp: op);
4687 if (mlir::acc::hasParDimsAttr(op) && isa<scf::ParallelOp>(Val: op)) {
4688 // parallel loops require special processing based on parallel dimension
4689 // this is mutually recursive with processOp
4690 scf::ParallelOp parallelOp = cast<scf::ParallelOp>(Val: op);
4691 processParallelOp(parallelOp);
4692 } else if (scf::ForOp seqLoop = dyn_cast<scf::ForOp>(Val: op)) {
4693 processSeqLoop(loopOp: seqLoop);
4694 } else if (acc::PrivatizeOp privatize = dyn_cast<acc::PrivatizeOp>(Val: op)) {
4695 processPrivatize(privatize);
4696 } else if (acc::PrivateLocalOp privateLocal =
4697 dyn_cast<acc::PrivateLocalOp>(Val: op)) {
4698 processPrivateLocal(privateLocal);
4699 } else if (acc::PredicateRegionOp predicateRegionOp =
4700 dyn_cast<acc::PredicateRegionOp>(Val: op)) {
4701 processPredicateRegion(interOp: predicateRegionOp);
4702 } else if (acc::ReductionAccumulateOp accumulateOp =
4703 dyn_cast<acc::ReductionAccumulateOp>(Val: op)) {
4704 processAccumulateOp(op: accumulateOp);
4705 } else if (auto accumulateArrayOp =
4706 dyn_cast<acc::ReductionAccumulateArrayOp>(Val: op)) {
4707 processAccumulateArrayOp(op: accumulateArrayOp);
4708 } else if (acc::ReductionInitOp reductionInitOp =
4709 dyn_cast<acc::ReductionInitOp>(Val: op)) {
4710 processReductionOp(op: reductionInitOp);
4711 } else if (acc::ReductionCombineOp reductionCombineOp =
4712 dyn_cast<acc::ReductionCombineOp>(Val: op)) {
4713 processReductionCombineOp(op: reductionCombineOp);
4714 } else if (auto combineRegionOp =
4715 dyn_cast<acc::ReductionCombineRegionOp>(Val: op)) {
4716 processCombineRegionOp(op: combineRegionOp);
4717 } else if (acc::ReductionOp accReductionOp = dyn_cast<acc::ReductionOp>(Val: op)) {
4718 mapping.map(from: accReductionOp->getResult(idx: 0), to: accReductionOp.getVarPtr());
4719 } else if (mapping.contains(from: op)) {
4720 // do nothing, operation in mapping signals it is already taken care of
4721 LLVM_DEBUG(llvm::dbgs() << "skipping mapped op: " << *op << "\n");
4722 } else if (isa<acc::YieldOp>(Val: op)) {
4723 for (auto [operand, result] :
4724 llvm::zip(t: op->getOperands(), u: op->getParentOp()->getResults())) {
4725 result.replaceAllUsesWith(newValue: mapping.lookup(from: operand));
4726 }
4727 } else if (isa<scf::ExecuteRegionOp>(Val: op)) {
4728 processExecuteRegion(op: cast<scf::ExecuteRegionOp>(Val: op));
4729 } else if (op->getNumRegions() == 0 ||
4730 isa<acc::OpenACCDialect>(Val: op->getDialect())) {
4731 processGenericOp(op);
4732 } else {
4733 processGenericOpWithRegions(op);
4734 }
4735}
4736
4737/// Fold `acc.par_width` to its launch operand or constant one.
4738class RemoveParWidth : public OpRewritePattern<acc::ParWidthOp> {
4739 using OpRewritePattern<acc::ParWidthOp>::OpRewritePattern;
4740 LogicalResult matchAndRewrite(acc::ParWidthOp op,
4741 PatternRewriter &rewriter) const override {
4742 if (Value launchArg = op.getLaunchArg()) {
4743 rewriter.replaceOp(op, newValues: launchArg);
4744 } else {
4745 Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: op.getLoc(), value: 1);
4746 rewriter.replaceOp(op, newValues: one);
4747 }
4748 return success();
4749 }
4750};
4751
4752/// Rewrite pattern that lowers `acc.compute_region` via ACCCGToGPULowering.
4753class ACCComputeRegionToGPUPattern
4754 : public OpRewritePattern<acc::ComputeRegionOp> {
4755public:
4756 ACCComputeRegionToGPUPattern(MLIRContext *context,
4757 acc::OpenACCSupport &accSupport,
4758 const ACCCGToGPUOptions &options)
4759 : OpRewritePattern<acc::ComputeRegionOp>(context), accSupport(accSupport),
4760 options(options) {}
4761
4762 LogicalResult matchAndRewrite(acc::ComputeRegionOp op,
4763 PatternRewriter &rewriter) const override {
4764 ACCCGToGPULowering kernelOpRewriter(op, rewriter, accSupport, options);
4765 return kernelOpRewriter.rewrite();
4766 }
4767
4768private:
4769 acc::OpenACCSupport &accSupport;
4770 const ACCCGToGPUOptions &options;
4771};
4772
4773class ACCCGToGPU : public acc::impl::ACCCGToGPUBase<ACCCGToGPU> {
4774public:
4775 using acc::impl::ACCCGToGPUBase<ACCCGToGPU>::ACCCGToGPUBase;
4776
4777 void getDependentDialects(DialectRegistry &registry) const override {
4778 acc::impl::ACCCGToGPUBase<ACCCGToGPU>::getDependentDialects(registry);
4779 acc::registerAllExtensions(registry);
4780 }
4781
4782 void runOnOperation() override {
4783 FunctionOpInterface funcOp = getOperation();
4784 MLIRContext *context = funcOp->getContext();
4785
4786 assert(deviceType != mlir::acc::DeviceType::Host &&
4787 deviceType != mlir::acc::DeviceType::Multicore &&
4788 "ACCCGToGPU only supports GPU device types");
4789
4790 // Validate parallel-loop mappings before any compute region is rewritten.
4791 // Loops outside compute regions do not require GPU mappings.
4792 WalkResult result = funcOp->walk(callback: [](scf::ParallelOp loop) {
4793 if (!loop->getParentOfType<acc::ComputeRegionOp>() ||
4794 mlir::acc::hasParDimsAttr(op: loop))
4795 return WalkResult::advance();
4796 loop.emitOpError(message: "requires an 'acc.par_dims' attribute");
4797 return WalkResult::interrupt();
4798 });
4799 if (result.wasInterrupted()) {
4800 signalPassFailure();
4801 return;
4802 }
4803
4804 ACCCGToGPUOptions options;
4805 options.deviceType = deviceType;
4806 options.maxWorkgroupSharedMemory = maxWorkgroupSharedMemory;
4807 options.maxThreadPrivateStack = maxThreadPrivateStack;
4808 options.subgroupSize = subgroupSize;
4809
4810 // Try to get cached parent analysis first, fall back to local analysis.
4811 std::optional<std::reference_wrapper<acc::OpenACCSupport>> cachedAnalysis =
4812 getCachedParentAnalysis<acc::OpenACCSupport>(parent: funcOp->getParentOp());
4813 acc::OpenACCSupport &accSupport = cachedAnalysis
4814 ? cachedAnalysis->get()
4815 : getAnalysis<acc::OpenACCSupport>();
4816
4817 RewritePatternSet patterns(context);
4818 patterns.insert<ACCComputeRegionToGPUPattern>(arg&: context, args&: accSupport, args&: options);
4819 patterns.insert<RemoveParWidth>(arg&: context);
4820 ConversionTarget target(getContext());
4821 target.markUnknownOpDynamicallyLegal(fn: [](Operation *) { return true; });
4822 target.addIllegalOp<acc::ComputeRegionOp, acc::ParWidthOp>();
4823 if (failed(Result: applyPartialConversion(op: getOperation(), target,
4824 patterns: std::move(patterns)))) {
4825 signalPassFailure();
4826 }
4827 }
4828};
4829} // namespace
4830