1//===- SubgroupReduceToDPP.cpp - subgroup_reduce to AMDGPU DPP lowering ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implements lowering of `gpu.subgroup_reduce` ops to AMDGPU DPP operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h"
14
15#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/GPU/IR/GPUDialect.h"
18#include "mlir/Dialect/GPU/Utils/GPUUtils.h"
19#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
20#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/IR/PatternMatch.h"
23#include "llvm/Support/FormatVariadic.h"
24#include <cassert>
25
26using namespace mlir;
27
28namespace {
29static FailureOr<Value>
30createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op,
31 Value input, gpu::AllReduceOperation mode,
32 const gpu::SubgroupReduceClusterInfo &ci,
33 const ROCDL::TargetInfo &target) {
34 Location loc = op.getLoc();
35 Value dpp;
36 Value res = input;
37 constexpr int allRows = 0xf;
38 constexpr int allBanks = 0xf;
39 const bool boundCtrl = true;
40 if (ci.clusterSize >= 2) {
41 // Perform reduction between all lanes N <-> N+1.
42 dpp = amdgpu::DPPOp::create(
43 builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::quad_perm,
44 permArgument: rewriter.getI32ArrayAttr(values: {1, 0, 3, 2}), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl);
45 res = vector::makeArithReduction(b&: rewriter, loc,
46 kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
47 }
48
49 if (ci.clusterSize >= 4) {
50 // Perform reduction between all lanes N <-> N+2.
51 dpp = amdgpu::DPPOp::create(
52 builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::quad_perm,
53 permArgument: rewriter.getI32ArrayAttr(values: {2, 3, 0, 1}), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl);
54 res = vector::makeArithReduction(b&: rewriter, loc,
55 kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
56 }
57 if (ci.clusterSize >= 8) {
58 // Perform reduction between all lanes N <-> 7-N,
59 // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4].
60 dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res,
61 kind: amdgpu::DPPPerm::row_half_mirror,
62 permArgument: rewriter.getUnitAttr(), row_mask: allRows, bank_mask: allBanks,
63 bound_ctrl: boundCtrl);
64 res = vector::makeArithReduction(b&: rewriter, loc,
65 kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
66 }
67 if (ci.clusterSize >= 16) {
68 // Perform reduction between all lanes N <-> 15-N,
69 // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8].
70 dpp = amdgpu::DPPOp::create(
71 builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::row_mirror,
72 permArgument: rewriter.getUnitAttr(), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl);
73 res = vector::makeArithReduction(b&: rewriter, loc,
74 kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
75 }
76 if (ci.clusterSize >= 32) {
77 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX10_INSTS)) {
78 // Broadcast last value from each row to next row.
79 // Use row mask to avoid polluting row 0 (and row 2 if wave-64).
80 dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res,
81 kind: amdgpu::DPPPerm::row_bcast_15,
82 permArgument: rewriter.getUnitAttr(), row_mask: 0xa, bank_mask: allBanks,
83 /*bound_ctrl*/ false);
84 res = vector::makeArithReduction(
85 b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
86
87 // For subgroupSize = 64, at this point lanes [16, 32) contain the full
88 // reduction over lanes [0, 32), but lanes [0, 16) do not. Similarly,
89 // lanes [48, 64) contain the full reduction over lanes [32, 64), but
90 // lanes [32, 48) do not.
91 //
92 // If subgroup size is 64 and cluster size is 64, we don't need lanes [0,
93 // 16) and [32, 48) to have the correct cluster-32 reduction values at
94 // this point, because only lane 63's value will ultimately be read in
95 // this full-cluster case.
96 //
97 // If subgroup size is 64 and cluster size is 32, we need to ensure that
98 // lanes [0, 16) and [32, 48) have the correct final cluster-32 reduction
99 // values (subgroup_reduce guarantees that all lanes within each cluster
100 // contain the final reduction value). We do this by broadcasting lane
101 // 31's value to lanes [0, 16) and lanes 63's value to lanes [32, 48).
102 //
103 // See https://gpuopen.com/learn/amd-gcn-assembly-cross-lane-operations
104 // for an illustration of how this within-cluster broadcast works with a
105 // swizzle.
106 if (ci.subgroupSize == 64 && ci.clusterSize == 32) {
107 res =
108 amdgpu::SwizzleBitModeOp::create(builder&: rewriter, location: loc, src: res, /*and_mask=*/0,
109 /*or_mask=*/31,
110 /*xor_mask=*/0);
111 }
112 } else if (!target.has(feature: llvm::AMDGPU::FEAT_GFX13_INSTS)) {
113 // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2).
114 Value uint32Max = arith::ConstantOp::create(
115 builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: -1));
116 dpp = ROCDL::PermlaneX16Op::create(builder&: rewriter, location: loc, res: res.getType(), old: res, src0: res,
117 src1: uint32Max, src2: uint32Max,
118 /*fi=*/true,
119 /*boundControl=*/false);
120 res = vector::makeArithReduction(
121 b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: res, acc: dpp);
122 } else {
123 return rewriter.notifyMatchFailure(
124 arg&: op, msg: "Subgroup reduce lowering to DPP not currently supported for "
125 "this device.");
126 }
127 if (ci.subgroupSize == 32) {
128 Value lane31 = arith::ConstantOp::create(
129 builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 31));
130 res =
131 ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane31);
132 }
133 }
134 if (ci.clusterSize >= 64) {
135 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX10_INSTS)) {
136 // Broadcast 31st lane value to rows 2 and 3.
137 dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res,
138 kind: amdgpu::DPPPerm::row_bcast_31,
139 permArgument: rewriter.getUnitAttr(), row_mask: 0xf, bank_mask: allBanks,
140 /*bound_ctrl*/ true);
141 res = vector::makeArithReduction(
142 b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: dpp, acc: res);
143 // Obtain reduction from last rows, the previous rows are polluted.
144 Value lane63 = arith::ConstantOp::create(
145 builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 63));
146 res =
147 ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane63);
148
149 } else if (!target.has(feature: llvm::AMDGPU::FEAT_GFX13_INSTS)) {
150 // Assume reduction across 32 lanes has been done.
151 // Perform final reduction manually by summing values in lane 0 and
152 // lane 32.
153 Value lane31 = arith::ConstantOp::create(
154 builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 31));
155 Value lane63 = arith::ConstantOp::create(
156 builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 63));
157 lane31 =
158 ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane31);
159 lane63 =
160 ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane63);
161 res = vector::makeArithReduction(
162 b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: lane31, acc: lane63);
163 } else {
164 return rewriter.notifyMatchFailure(
165 arg&: op, msg: "Subgroup reduce lowering to DPP not currently supported for "
166 "this device.");
167 }
168 }
169 assert(res.getType() == input.getType());
170 return res;
171}
172
173/// Collect a set of patterns to lower `gpu.subgroup_reduce` into `amdgpu.dpp`
174/// ops over scalar types. Assumes that the subgroup has
175/// `subgroupSize` lanes. Applicable only to AMD GPUs.
176struct ScalarSubgroupReduceToDPP final
177 : OpRewritePattern<gpu::SubgroupReduceOp> {
178 ScalarSubgroupReduceToDPP(MLIRContext *ctx, unsigned subgroupSize,
179 bool matchClustered,
180 const ROCDL::TargetInfo &target,
181 PatternBenefit benefit)
182 : OpRewritePattern(ctx, benefit), subgroupSize(subgroupSize),
183 matchClustered(matchClustered), target(target) {}
184
185 LogicalResult matchAndRewrite(gpu::SubgroupReduceOp op,
186 PatternRewriter &rewriter) const override {
187 if (op.getClusterSize().has_value() != matchClustered) {
188 return rewriter.notifyMatchFailure(
189 arg&: op, msg: llvm::formatv(Fmt: "op is {0}clustered but pattern is configured to "
190 "only match {1}clustered ops",
191 Vals: matchClustered ? "non-" : "",
192 Vals: matchClustered ? "" : "non-"));
193 }
194 auto ci = gpu::getSubgroupReduceClusterInfo(op, subgroupSize);
195 if (failed(Result: ci))
196 return failure();
197
198 if (ci->clusterStride != 1)
199 return rewriter.notifyMatchFailure(
200 arg&: op, msg: "Subgroup reductions using DPP are currently only available for "
201 "clusters of contiguous lanes.");
202
203 Type valueTy = op.getType();
204 if (!valueTy.isIntOrFloat())
205 return rewriter.notifyMatchFailure(
206 arg&: op, msg: "Value type is not a compatible scalar.");
207
208 FailureOr<Value> dpp = createSubgroupDPPReduction(
209 rewriter, op, input: op.getValue(), mode: op.getOp(), ci: *ci, target);
210 if (failed(Result: dpp))
211 return failure();
212
213 rewriter.replaceOp(op, newValues: dpp.value());
214 return success();
215 }
216
217private:
218 unsigned subgroupSize = 0;
219 bool matchClustered = false;
220 ROCDL::TargetInfo target;
221};
222} // namespace
223
224void mlir::populateGpuLowerSubgroupReduceToDPPPatterns(
225 RewritePatternSet &patterns, unsigned subgroupSize,
226 const ROCDL::TargetInfo &target, PatternBenefit benefit) {
227 patterns.add<ScalarSubgroupReduceToDPP>(arg: patterns.getContext(), args&: subgroupSize,
228 /*matchClustered=*/args: false, args: target,
229 args&: benefit);
230}
231
232void mlir::populateGpuLowerClusteredSubgroupReduceToDPPPatterns(
233 RewritePatternSet &patterns, unsigned subgroupSize,
234 const ROCDL::TargetInfo &target, PatternBenefit benefit) {
235 patterns.add<ScalarSubgroupReduceToDPP>(arg: patterns.getContext(), args&: subgroupSize,
236 /*matchClustered=*/args: true, args: target,
237 args&: benefit);
238}
239