| 1 | //===- SubgroupReduceToDPP.cpp - subgroup_reduce to AMDGPU DPP lowering ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Implements lowering of `gpu.subgroup_reduce` ops to AMDGPU DPP operations. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h" |
| 14 | |
| 15 | #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h" |
| 16 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 17 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 18 | #include "mlir/Dialect/GPU/Utils/GPUUtils.h" |
| 19 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 20 | #include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h" |
| 21 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 22 | #include "mlir/IR/PatternMatch.h" |
| 23 | #include "llvm/Support/FormatVariadic.h" |
| 24 | #include <cassert> |
| 25 | |
| 26 | using namespace mlir; |
| 27 | |
| 28 | namespace { |
| 29 | static FailureOr<Value> |
| 30 | createSubgroupDPPReduction(PatternRewriter &rewriter, gpu::SubgroupReduceOp op, |
| 31 | Value input, gpu::AllReduceOperation mode, |
| 32 | const gpu::SubgroupReduceClusterInfo &ci, |
| 33 | const ROCDL::TargetInfo &target) { |
| 34 | Location loc = op.getLoc(); |
| 35 | Value dpp; |
| 36 | Value res = input; |
| 37 | constexpr int allRows = 0xf; |
| 38 | constexpr int allBanks = 0xf; |
| 39 | const bool boundCtrl = true; |
| 40 | if (ci.clusterSize >= 2) { |
| 41 | // Perform reduction between all lanes N <-> N+1. |
| 42 | dpp = amdgpu::DPPOp::create( |
| 43 | builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::quad_perm, |
| 44 | permArgument: rewriter.getI32ArrayAttr(values: {1, 0, 3, 2}), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl); |
| 45 | res = vector::makeArithReduction(b&: rewriter, loc, |
| 46 | kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 47 | } |
| 48 | |
| 49 | if (ci.clusterSize >= 4) { |
| 50 | // Perform reduction between all lanes N <-> N+2. |
| 51 | dpp = amdgpu::DPPOp::create( |
| 52 | builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::quad_perm, |
| 53 | permArgument: rewriter.getI32ArrayAttr(values: {2, 3, 0, 1}), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl); |
| 54 | res = vector::makeArithReduction(b&: rewriter, loc, |
| 55 | kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 56 | } |
| 57 | if (ci.clusterSize >= 8) { |
| 58 | // Perform reduction between all lanes N <-> 7-N, |
| 59 | // e.g lane[0] <-> lane[7], lane[1] <-> lane[6]..., lane[3] <-> lane[4]. |
| 60 | dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, |
| 61 | kind: amdgpu::DPPPerm::row_half_mirror, |
| 62 | permArgument: rewriter.getUnitAttr(), row_mask: allRows, bank_mask: allBanks, |
| 63 | bound_ctrl: boundCtrl); |
| 64 | res = vector::makeArithReduction(b&: rewriter, loc, |
| 65 | kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 66 | } |
| 67 | if (ci.clusterSize >= 16) { |
| 68 | // Perform reduction between all lanes N <-> 15-N, |
| 69 | // e.g lane[0] <-> lane[15], lane[1] <-> lane[14]..., lane[7] <-> lane[8]. |
| 70 | dpp = amdgpu::DPPOp::create( |
| 71 | builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, kind: amdgpu::DPPPerm::row_mirror, |
| 72 | permArgument: rewriter.getUnitAttr(), row_mask: allRows, bank_mask: allBanks, bound_ctrl: boundCtrl); |
| 73 | res = vector::makeArithReduction(b&: rewriter, loc, |
| 74 | kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 75 | } |
| 76 | if (ci.clusterSize >= 32) { |
| 77 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX10_INSTS)) { |
| 78 | // Broadcast last value from each row to next row. |
| 79 | // Use row mask to avoid polluting row 0 (and row 2 if wave-64). |
| 80 | dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, |
| 81 | kind: amdgpu::DPPPerm::row_bcast_15, |
| 82 | permArgument: rewriter.getUnitAttr(), row_mask: 0xa, bank_mask: allBanks, |
| 83 | /*bound_ctrl*/ false); |
| 84 | res = vector::makeArithReduction( |
| 85 | b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 86 | |
| 87 | // For subgroupSize = 64, at this point lanes [16, 32) contain the full |
| 88 | // reduction over lanes [0, 32), but lanes [0, 16) do not. Similarly, |
| 89 | // lanes [48, 64) contain the full reduction over lanes [32, 64), but |
| 90 | // lanes [32, 48) do not. |
| 91 | // |
| 92 | // If subgroup size is 64 and cluster size is 64, we don't need lanes [0, |
| 93 | // 16) and [32, 48) to have the correct cluster-32 reduction values at |
| 94 | // this point, because only lane 63's value will ultimately be read in |
| 95 | // this full-cluster case. |
| 96 | // |
| 97 | // If subgroup size is 64 and cluster size is 32, we need to ensure that |
| 98 | // lanes [0, 16) and [32, 48) have the correct final cluster-32 reduction |
| 99 | // values (subgroup_reduce guarantees that all lanes within each cluster |
| 100 | // contain the final reduction value). We do this by broadcasting lane |
| 101 | // 31's value to lanes [0, 16) and lanes 63's value to lanes [32, 48). |
| 102 | // |
| 103 | // See https://gpuopen.com/learn/amd-gcn-assembly-cross-lane-operations |
| 104 | // for an illustration of how this within-cluster broadcast works with a |
| 105 | // swizzle. |
| 106 | if (ci.subgroupSize == 64 && ci.clusterSize == 32) { |
| 107 | res = |
| 108 | amdgpu::SwizzleBitModeOp::create(builder&: rewriter, location: loc, src: res, /*and_mask=*/0, |
| 109 | /*or_mask=*/31, |
| 110 | /*xor_mask=*/0); |
| 111 | } |
| 112 | } else if (!target.has(feature: llvm::AMDGPU::FEAT_GFX13_INSTS)) { |
| 113 | // Use a permute lane to cross rows (row 1 <-> row 0, row 3 <-> row 2). |
| 114 | Value uint32Max = arith::ConstantOp::create( |
| 115 | builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: -1)); |
| 116 | dpp = ROCDL::PermlaneX16Op::create(builder&: rewriter, location: loc, res: res.getType(), old: res, src0: res, |
| 117 | src1: uint32Max, src2: uint32Max, |
| 118 | /*fi=*/true, |
| 119 | /*boundControl=*/false); |
| 120 | res = vector::makeArithReduction( |
| 121 | b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: res, acc: dpp); |
| 122 | } else { |
| 123 | return rewriter.notifyMatchFailure( |
| 124 | arg&: op, msg: "Subgroup reduce lowering to DPP not currently supported for " |
| 125 | "this device." ); |
| 126 | } |
| 127 | if (ci.subgroupSize == 32) { |
| 128 | Value lane31 = arith::ConstantOp::create( |
| 129 | builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 31)); |
| 130 | res = |
| 131 | ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane31); |
| 132 | } |
| 133 | } |
| 134 | if (ci.clusterSize >= 64) { |
| 135 | if (!target.has(feature: llvm::AMDGPU::FEAT_GFX10_INSTS)) { |
| 136 | // Broadcast 31st lane value to rows 2 and 3. |
| 137 | dpp = amdgpu::DPPOp::create(builder&: rewriter, location: loc, result: res.getType(), old: res, src: res, |
| 138 | kind: amdgpu::DPPPerm::row_bcast_31, |
| 139 | permArgument: rewriter.getUnitAttr(), row_mask: 0xf, bank_mask: allBanks, |
| 140 | /*bound_ctrl*/ true); |
| 141 | res = vector::makeArithReduction( |
| 142 | b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: dpp, acc: res); |
| 143 | // Obtain reduction from last rows, the previous rows are polluted. |
| 144 | Value lane63 = arith::ConstantOp::create( |
| 145 | builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 63)); |
| 146 | res = |
| 147 | ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane63); |
| 148 | |
| 149 | } else if (!target.has(feature: llvm::AMDGPU::FEAT_GFX13_INSTS)) { |
| 150 | // Assume reduction across 32 lanes has been done. |
| 151 | // Perform final reduction manually by summing values in lane 0 and |
| 152 | // lane 32. |
| 153 | Value lane31 = arith::ConstantOp::create( |
| 154 | builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 31)); |
| 155 | Value lane63 = arith::ConstantOp::create( |
| 156 | builder&: rewriter, location: loc, result: rewriter.getI32Type(), value: rewriter.getI32IntegerAttr(value: 63)); |
| 157 | lane31 = |
| 158 | ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane31); |
| 159 | lane63 = |
| 160 | ROCDL::ReadlaneOp::create(builder&: rewriter, location: loc, res: res.getType(), src0: res, src1: lane63); |
| 161 | res = vector::makeArithReduction( |
| 162 | b&: rewriter, loc, kind: gpu::convertReductionKind(mode), v1: lane31, acc: lane63); |
| 163 | } else { |
| 164 | return rewriter.notifyMatchFailure( |
| 165 | arg&: op, msg: "Subgroup reduce lowering to DPP not currently supported for " |
| 166 | "this device." ); |
| 167 | } |
| 168 | } |
| 169 | assert(res.getType() == input.getType()); |
| 170 | return res; |
| 171 | } |
| 172 | |
| 173 | /// Collect a set of patterns to lower `gpu.subgroup_reduce` into `amdgpu.dpp` |
| 174 | /// ops over scalar types. Assumes that the subgroup has |
| 175 | /// `subgroupSize` lanes. Applicable only to AMD GPUs. |
| 176 | struct ScalarSubgroupReduceToDPP final |
| 177 | : OpRewritePattern<gpu::SubgroupReduceOp> { |
| 178 | ScalarSubgroupReduceToDPP(MLIRContext *ctx, unsigned subgroupSize, |
| 179 | bool matchClustered, |
| 180 | const ROCDL::TargetInfo &target, |
| 181 | PatternBenefit benefit) |
| 182 | : OpRewritePattern(ctx, benefit), subgroupSize(subgroupSize), |
| 183 | matchClustered(matchClustered), target(target) {} |
| 184 | |
| 185 | LogicalResult matchAndRewrite(gpu::SubgroupReduceOp op, |
| 186 | PatternRewriter &rewriter) const override { |
| 187 | if (op.getClusterSize().has_value() != matchClustered) { |
| 188 | return rewriter.notifyMatchFailure( |
| 189 | arg&: op, msg: llvm::formatv(Fmt: "op is {0}clustered but pattern is configured to " |
| 190 | "only match {1}clustered ops" , |
| 191 | Vals: matchClustered ? "non-" : "" , |
| 192 | Vals: matchClustered ? "" : "non-" )); |
| 193 | } |
| 194 | auto ci = gpu::getSubgroupReduceClusterInfo(op, subgroupSize); |
| 195 | if (failed(Result: ci)) |
| 196 | return failure(); |
| 197 | |
| 198 | if (ci->clusterStride != 1) |
| 199 | return rewriter.notifyMatchFailure( |
| 200 | arg&: op, msg: "Subgroup reductions using DPP are currently only available for " |
| 201 | "clusters of contiguous lanes." ); |
| 202 | |
| 203 | Type valueTy = op.getType(); |
| 204 | if (!valueTy.isIntOrFloat()) |
| 205 | return rewriter.notifyMatchFailure( |
| 206 | arg&: op, msg: "Value type is not a compatible scalar." ); |
| 207 | |
| 208 | FailureOr<Value> dpp = createSubgroupDPPReduction( |
| 209 | rewriter, op, input: op.getValue(), mode: op.getOp(), ci: *ci, target); |
| 210 | if (failed(Result: dpp)) |
| 211 | return failure(); |
| 212 | |
| 213 | rewriter.replaceOp(op, newValues: dpp.value()); |
| 214 | return success(); |
| 215 | } |
| 216 | |
| 217 | private: |
| 218 | unsigned subgroupSize = 0; |
| 219 | bool matchClustered = false; |
| 220 | ROCDL::TargetInfo target; |
| 221 | }; |
| 222 | } // namespace |
| 223 | |
| 224 | void mlir::populateGpuLowerSubgroupReduceToDPPPatterns( |
| 225 | RewritePatternSet &patterns, unsigned subgroupSize, |
| 226 | const ROCDL::TargetInfo &target, PatternBenefit benefit) { |
| 227 | patterns.add<ScalarSubgroupReduceToDPP>(arg: patterns.getContext(), args&: subgroupSize, |
| 228 | /*matchClustered=*/args: false, args: target, |
| 229 | args&: benefit); |
| 230 | } |
| 231 | |
| 232 | void mlir::populateGpuLowerClusteredSubgroupReduceToDPPPatterns( |
| 233 | RewritePatternSet &patterns, unsigned subgroupSize, |
| 234 | const ROCDL::TargetInfo &target, PatternBenefit benefit) { |
| 235 | patterns.add<ScalarSubgroupReduceToDPP>(arg: patterns.getContext(), args&: subgroupSize, |
| 236 | /*matchClustered=*/args: true, args: target, |
| 237 | args&: benefit); |
| 238 | } |
| 239 | |