1//===- SubgroupIdRewriter.cpp - Implementation of SubgroupId rewriting ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements in-dialect rewriting of the gpu.subgroup_id op for archs
10// where:
11// subgroup_id = (tid.x + dim.x * (tid.y + dim.y * tid.z)) / subgroup_size
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/GPU/IR/GPUDialect.h"
17#include "mlir/Dialect/GPU/Transforms/Passes.h"
18#include "mlir/Dialect/Index/IR/IndexOps.h"
19#include "mlir/IR/Builders.h"
20#include "mlir/IR/PatternMatch.h"
21
22using namespace mlir;
23
24namespace {
25struct GpuSubgroupIdRewriter final : OpRewritePattern<gpu::SubgroupIdOp> {
26 using OpRewritePattern<gpu::SubgroupIdOp>::OpRewritePattern;
27
28 LogicalResult matchAndRewrite(gpu::SubgroupIdOp op,
29 PatternRewriter &rewriter) const override {
30 // Calculation of the thread's subgroup identifier.
31 //
32 // The process involves mapping the thread's 3D identifier within its
33 // block (b_id.x, b_id.y, b_id.z) to a 1D linear index.
34 // This linearization assumes a layout where the x-dimension (w_dim.x)
35 // varies most rapidly (i.e., it is the innermost dimension).
36 //
37 // The formula for the linearized thread index is:
38 // L = tid.x + dim.x * (tid.y + (dim.y * tid.z))
39 //
40 // Subsequently, the range of linearized indices [0, N_threads-1] is
41 // divided into consecutive, non-overlapping segments, each representing
42 // a subgroup of size 'subgroup_size'.
43 //
44 // Example Partitioning (N = subgroup_size):
45 // | Subgroup 0 | Subgroup 1 | Subgroup 2 | ... |
46 // | Indices 0..N-1 | Indices N..2N-1 | Indices 2N..3N-1| ... |
47 //
48 // The subgroup identifier is obtained via integer division of the
49 // linearized thread index by the predefined 'subgroup_size'.
50 //
51 // subgroup_id = floor( L / subgroup_size )
52 // = (tid.x + dim.x * (tid.y + dim.y * tid.z)) /
53 // subgroup_size
54
55 Location loc = op->getLoc();
56 Type indexType = rewriter.getIndexType();
57
58 auto asMaybeIndexAttr = [&](std::optional<uint32_t> bound) -> IntegerAttr {
59 if (!bound)
60 return IntegerAttr();
61 return IntegerAttr::get(
62 type: indexType, value: static_cast<int64_t>(static_cast<uint64_t>(*bound)));
63 };
64
65 IntegerAttr maybeKnownDimX =
66 asMaybeIndexAttr(gpu::getKnownDimensionSizeAround(
67 op, kind: gpu::DimensionKind::Block, dim: gpu::Dimension::x));
68 IntegerAttr maybeKnownDimY =
69 asMaybeIndexAttr(gpu::getKnownDimensionSizeAround(
70 op, kind: gpu::DimensionKind::Block, dim: gpu::Dimension::y));
71 IntegerAttr maybeKnownDimZ =
72 asMaybeIndexAttr(gpu::getKnownDimensionSizeAround(
73 op, kind: gpu::DimensionKind::Block, dim: gpu::Dimension::z));
74
75 Value dimX, dimY;
76 if (maybeKnownDimX)
77 dimX = arith::ConstantOp::create(builder&: rewriter, location: loc, value: maybeKnownDimX);
78 else
79 dimX = gpu::BlockDimOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x);
80 if (maybeKnownDimY)
81 dimY = arith::ConstantOp::create(builder&: rewriter, location: loc, value: maybeKnownDimY);
82 else
83 dimY = gpu::BlockDimOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::y);
84
85 Value tidX = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x,
86 upper_bound: maybeKnownDimX);
87 Value tidY = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::y,
88 upper_bound: maybeKnownDimY);
89 Value tidZ = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::z,
90 upper_bound: maybeKnownDimZ);
91
92 // Block dimensions don't exceed a signed int32_t maximum, and neither does
93 // their product, on any realistic hardware, nor would any targets compile
94 // with index < 32 bits, so we can assert no overflow.
95 auto flags =
96 arith::IntegerOverflowFlags::nsw | arith::IntegerOverflowFlags::nuw;
97 Value dimYxIdZ =
98 arith::MulIOp::create(builder&: rewriter, location: loc, result: indexType, lhs: dimY, rhs: tidZ, overflowFlags: flags);
99 Value dimYxIdZPlusIdY =
100 arith::AddIOp::create(builder&: rewriter, location: loc, result: indexType, lhs: dimYxIdZ, rhs: tidY, overflowFlags: flags);
101 Value dimYxIdZPlusIdYTimesDimX = arith::MulIOp::create(
102 builder&: rewriter, location: loc, result: indexType, lhs: dimX, rhs: dimYxIdZPlusIdY, overflowFlags: flags);
103 Value idXPlusDimYxIdZPlusIdYTimesDimX = arith::AddIOp::create(
104 builder&: rewriter, location: loc, result: indexType, lhs: tidX, rhs: dimYxIdZPlusIdYTimesDimX, overflowFlags: flags);
105 Value subgroupSize = gpu::SubgroupSizeOp::create(
106 builder&: rewriter, location: loc, result: rewriter.getIndexType(), /*upper_bound = */ nullptr);
107 Value subgroupIdOp =
108 arith::DivUIOp::create(builder&: rewriter, location: loc, result: indexType,
109 lhs: idXPlusDimYxIdZPlusIdYTimesDimX, rhs: subgroupSize);
110 rewriter.replaceOp(op, newValues: {subgroupIdOp});
111 return success();
112 }
113};
114
115} // namespace
116
117void mlir::populateGpuSubgroupIdPatterns(RewritePatternSet &patterns) {
118 patterns.add<GpuSubgroupIdRewriter>(arg: patterns.getContext());
119}
120