| 1 | //===- InferIntRangeInterfaceImpls.cpp - Integer range impls for gpu -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 10 | #include "mlir/IR/Matchers.h" |
| 11 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 12 | #include "mlir/Interfaces/InferIntRangeInterface.h" |
| 13 | #include "mlir/Interfaces/Utils/InferIntRangeCommon.h" |
| 14 | #include <optional> |
| 15 | |
| 16 | using namespace mlir; |
| 17 | using namespace mlir::gpu; |
| 18 | |
| 19 | // Maximum grid and block dimensions of all known GPUs are less than 2^32. |
| 20 | static constexpr uint64_t kMaxDim = std::numeric_limits<uint32_t>::max(); |
| 21 | // Maximum cluster size. |
| 22 | static constexpr uint64_t kMaxClusterDim = 16; |
| 23 | // Maximum subgroups are no larger than 128. |
| 24 | static constexpr uint64_t kMaxSubgroupSize = 128; |
| 25 | |
| 26 | static ConstantIntRanges getIndexRange(uint64_t umin, uint64_t umax) { |
| 27 | unsigned width = IndexType::kInternalStorageBitWidth; |
| 28 | return ConstantIntRanges::fromUnsigned(umin: APInt(width, umin), |
| 29 | umax: APInt(width, umax)); |
| 30 | } |
| 31 | |
| 32 | static uint64_t zext(uint32_t arg) { return static_cast<uint64_t>(arg); } |
| 33 | |
| 34 | static Value valueByDim(KernelDim3 dims, Dimension dim) { |
| 35 | switch (dim) { |
| 36 | case Dimension::x: |
| 37 | return dims.x; |
| 38 | case Dimension::y: |
| 39 | return dims.y; |
| 40 | case Dimension::z: |
| 41 | return dims.z; |
| 42 | } |
| 43 | llvm_unreachable("All dimension enum cases handled above" ); |
| 44 | } |
| 45 | |
| 46 | static std::optional<uint32_t> |
| 47 | getKnownLaunchAttr(GPUFuncOp func, DimensionKind dims, Dimension dim) { |
| 48 | DenseI32ArrayAttr bounds; |
| 49 | switch (dims) { |
| 50 | case DimensionKind::Other: |
| 51 | return std::nullopt; |
| 52 | case DimensionKind::Block: |
| 53 | bounds = func.getKnownBlockSizeAttr(); |
| 54 | break; |
| 55 | case DimensionKind::Grid: |
| 56 | bounds = func.getKnownGridSizeAttr(); |
| 57 | break; |
| 58 | case DimensionKind::Cluster: |
| 59 | bounds = func.getKnownClusterSizeAttr(); |
| 60 | break; |
| 61 | } |
| 62 | if (!bounds) |
| 63 | return std::nullopt; |
| 64 | if (bounds.size() <= static_cast<uint32_t>(dim)) |
| 65 | return std::nullopt; |
| 66 | return bounds[static_cast<uint32_t>(dim)]; |
| 67 | } |
| 68 | |
| 69 | static std::optional<uint32_t> getKnownLaunchAttr(FunctionOpInterface func, |
| 70 | StringRef attrName, |
| 71 | Dimension dim) { |
| 72 | auto bounds = |
| 73 | func.getOperation()->getDiscardableAttrOfType<DenseI32ArrayAttr>( |
| 74 | name: attrName); |
| 75 | if (!bounds) |
| 76 | return std::nullopt; |
| 77 | if (bounds.size() <= static_cast<uint32_t>(dim)) |
| 78 | return std::nullopt; |
| 79 | return bounds[static_cast<uint32_t>(dim)]; |
| 80 | } |
| 81 | |
| 82 | std::optional<uint32_t> |
| 83 | mlir::gpu::getKnownDimensionSizeAround(Operation *op, DimensionKind kind, |
| 84 | Dimension dim) { |
| 85 | if (auto launch = op->getParentOfType<LaunchOp>()) { |
| 86 | KernelDim3 bounds; |
| 87 | switch (kind) { |
| 88 | case DimensionKind::Other: |
| 89 | return std::nullopt; |
| 90 | case DimensionKind::Block: |
| 91 | bounds = launch.getBlockSizeOperandValues(); |
| 92 | break; |
| 93 | case DimensionKind::Grid: |
| 94 | bounds = launch.getGridSizeOperandValues(); |
| 95 | break; |
| 96 | case DimensionKind::Cluster: |
| 97 | if (launch.hasClusterSize()) { |
| 98 | auto clusterBounds = launch.getClusterSizeOperandValues(); |
| 99 | if (clusterBounds) |
| 100 | bounds = *clusterBounds; |
| 101 | } |
| 102 | break; |
| 103 | } |
| 104 | Value maybeBound = valueByDim(dims: bounds, dim); |
| 105 | APInt value; |
| 106 | if (maybeBound && matchPattern(value: maybeBound, pattern: m_ConstantInt(bind_value: &value))) |
| 107 | return value.getZExtValue(); |
| 108 | } |
| 109 | |
| 110 | if (auto gpuFunc = op->getParentOfType<GPUFuncOp>()) { |
| 111 | auto inherentAttr = getKnownLaunchAttr(func: gpuFunc, dims: kind, dim); |
| 112 | if (inherentAttr) |
| 113 | return inherentAttr; |
| 114 | } |
| 115 | if (auto func = op->getParentOfType<FunctionOpInterface>()) { |
| 116 | StringRef attrName; |
| 117 | switch (kind) { |
| 118 | case DimensionKind::Other: |
| 119 | return std::nullopt; |
| 120 | case DimensionKind::Block: |
| 121 | attrName = GPUDialect::KnownBlockSizeAttrHelper::getNameStr(); |
| 122 | break; |
| 123 | case DimensionKind::Grid: |
| 124 | attrName = GPUDialect::KnownGridSizeAttrHelper::getNameStr(); |
| 125 | break; |
| 126 | case DimensionKind::Cluster: |
| 127 | attrName = GPUDialect::KnownClusterSizeAttrHelper::getNameStr(); |
| 128 | break; |
| 129 | } |
| 130 | auto discardableAttr = getKnownLaunchAttr(func, attrName, dim); |
| 131 | if (discardableAttr) |
| 132 | return discardableAttr; |
| 133 | } |
| 134 | return std::nullopt; |
| 135 | } |
| 136 | |
| 137 | void ClusterDimOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 138 | SetIntRangeFn setResultRange) { |
| 139 | uint64_t max = kMaxDim; |
| 140 | if (auto specified = getUpperBound()) |
| 141 | max = specified->getZExtValue(); |
| 142 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 143 | } |
| 144 | |
| 145 | void ClusterDimBlocksOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 146 | SetIntRangeFn setResultRange) { |
| 147 | if (auto known = getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Cluster, |
| 148 | dim: getDimension())) |
| 149 | return setResultRange(getResult(), |
| 150 | getIndexRange(umin: zext(arg: *known), umax: zext(arg: *known))); |
| 151 | |
| 152 | uint64_t max = kMaxClusterDim; |
| 153 | if (auto specified = getUpperBound()) |
| 154 | max = specified->getZExtValue(); |
| 155 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 156 | } |
| 157 | |
| 158 | void ClusterIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 159 | SetIntRangeFn setResultRange) { |
| 160 | uint64_t max = kMaxDim; |
| 161 | if (auto specified = getUpperBound()) |
| 162 | max = specified->getZExtValue(); |
| 163 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 164 | } |
| 165 | |
| 166 | void ClusterBlockIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 167 | SetIntRangeFn setResultRange) { |
| 168 | uint64_t max = kMaxClusterDim; |
| 169 | if (auto known = getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Cluster, |
| 170 | dim: getDimension())) |
| 171 | max = zext(arg: *known); |
| 172 | if (auto specified = getUpperBound()) |
| 173 | max = specified->getZExtValue(); |
| 174 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 175 | } |
| 176 | |
| 177 | void BlockDimOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 178 | SetIntRangeFn setResultRange) { |
| 179 | std::optional<uint32_t> knownVal = |
| 180 | getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Block, dim: getDimension()); |
| 181 | if (knownVal) |
| 182 | return setResultRange(getResult(), |
| 183 | getIndexRange(umin: zext(arg: *knownVal), umax: zext(arg: *knownVal))); |
| 184 | |
| 185 | uint64_t max = kMaxDim; |
| 186 | if (auto specified = getUpperBound()) |
| 187 | max = specified->getZExtValue(); |
| 188 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 189 | } |
| 190 | |
| 191 | void BlockIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 192 | SetIntRangeFn setResultRange) { |
| 193 | uint64_t max = kMaxDim; |
| 194 | if (auto fromContext = getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Grid, |
| 195 | dim: getDimension())) |
| 196 | max = zext(arg: *fromContext); |
| 197 | if (auto specified = getUpperBound()) |
| 198 | max = specified->getZExtValue(); |
| 199 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 200 | } |
| 201 | |
| 202 | void GridDimOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 203 | SetIntRangeFn setResultRange) { |
| 204 | std::optional<uint32_t> knownVal = |
| 205 | getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Grid, dim: getDimension()); |
| 206 | if (knownVal) |
| 207 | return setResultRange(getResult(), |
| 208 | getIndexRange(umin: zext(arg: *knownVal), umax: zext(arg: *knownVal))); |
| 209 | uint64_t max = kMaxDim; |
| 210 | if (auto specified = getUpperBound()) |
| 211 | max = specified->getZExtValue(); |
| 212 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 213 | } |
| 214 | |
| 215 | void ThreadIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 216 | SetIntRangeFn setResultRange) { |
| 217 | uint64_t max = kMaxDim; |
| 218 | if (auto fromContext = getKnownDimensionSizeAround( |
| 219 | op: *this, kind: DimensionKind::Block, dim: getDimension())) |
| 220 | max = zext(arg: *fromContext); |
| 221 | if (auto specified = getUpperBound()) |
| 222 | max = specified->getZExtValue(); |
| 223 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 224 | } |
| 225 | |
| 226 | void LaneIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 227 | SetIntRangeFn setResultRange) { |
| 228 | uint64_t max = kMaxSubgroupSize; |
| 229 | if (auto specified = getUpperBound()) |
| 230 | max = specified->getZExtValue(); |
| 231 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 232 | } |
| 233 | |
| 234 | void SubgroupIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 235 | SetIntRangeFn setResultRange) { |
| 236 | uint64_t max = kMaxDim; |
| 237 | if (auto specified = getUpperBound()) |
| 238 | max = specified->getZExtValue(); |
| 239 | setResultRange(getResult(), getIndexRange(umin: 0, umax: max - 1ULL)); |
| 240 | } |
| 241 | |
| 242 | void GlobalIdOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 243 | SetIntRangeFn setResultRange) { |
| 244 | if (auto specified = getUpperBound()) |
| 245 | return setResultRange(getResult(), |
| 246 | getIndexRange(umin: 0, umax: specified->getZExtValue() - 1ULL)); |
| 247 | |
| 248 | uint64_t blockDimMax = zext( |
| 249 | arg: getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Block, dim: getDimension()) |
| 250 | .value_or(u: kMaxDim)); |
| 251 | uint64_t gridDimMax = zext( |
| 252 | arg: getKnownDimensionSizeAround(op: *this, kind: DimensionKind::Grid, dim: getDimension()) |
| 253 | .value_or(u: kMaxDim)); |
| 254 | setResultRange(getResult(), |
| 255 | getIndexRange(umin: 0, umax: (blockDimMax * gridDimMax) - 1ULL)); |
| 256 | } |
| 257 | |
| 258 | void NumSubgroupsOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 259 | SetIntRangeFn setResultRange) { |
| 260 | uint64_t max = kMaxDim; |
| 261 | if (auto specified = getUpperBound()) |
| 262 | max = specified->getZExtValue(); |
| 263 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 264 | } |
| 265 | |
| 266 | void SubgroupSizeOp::inferResultRanges(ArrayRef<ConstantIntRanges>, |
| 267 | SetIntRangeFn setResultRange) { |
| 268 | uint64_t max = kMaxSubgroupSize; |
| 269 | if (auto specified = getUpperBound()) |
| 270 | max = specified->getZExtValue(); |
| 271 | setResultRange(getResult(), getIndexRange(umin: 1, umax: max)); |
| 272 | } |
| 273 | |
| 274 | void LaunchOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 275 | SetIntRangeFn setResultRange) { |
| 276 | auto setRange = [&](const ConstantIntRanges &argRange, Value dimResult, |
| 277 | Value idxResult) { |
| 278 | if (argRange.umin().getBitWidth() != IndexType::kInternalStorageBitWidth) |
| 279 | return; |
| 280 | ConstantIntRanges dimRange = |
| 281 | argRange.intersection(other: getIndexRange(umin: 1, umax: kMaxDim)); |
| 282 | setResultRange(dimResult, dimRange); |
| 283 | ConstantIntRanges idxRange = |
| 284 | getIndexRange(umin: 0, umax: dimRange.umax().getZExtValue() - 1); |
| 285 | setResultRange(idxResult, idxRange); |
| 286 | }; |
| 287 | |
| 288 | argRanges = argRanges.drop_front(N: getAsyncDependencies().size()); |
| 289 | KernelDim3 gridDims = getGridSize(); |
| 290 | KernelDim3 blockIds = getBlockIds(); |
| 291 | setRange(argRanges[0], gridDims.x, blockIds.x); |
| 292 | setRange(argRanges[1], gridDims.y, blockIds.y); |
| 293 | setRange(argRanges[2], gridDims.z, blockIds.z); |
| 294 | KernelDim3 blockDims = getBlockSize(); |
| 295 | KernelDim3 threadIds = getThreadIds(); |
| 296 | setRange(argRanges[3], blockDims.x, threadIds.x); |
| 297 | setRange(argRanges[4], blockDims.y, threadIds.y); |
| 298 | setRange(argRanges[5], blockDims.z, threadIds.z); |
| 299 | |
| 300 | if (auto clusterIds = getClusterIds()) { |
| 301 | KernelDim3 clusterDims = *getClusterSize(); |
| 302 | auto setClusterRange = [&](unsigned dim, Value dimResult, Value idxResult) { |
| 303 | unsigned clusterOperandIndex = kNumConfigOperands + dim; |
| 304 | if (argRanges[dim].umin().getBitWidth() != |
| 305 | IndexType::kInternalStorageBitWidth || |
| 306 | argRanges[clusterOperandIndex].umin().getBitWidth() != |
| 307 | IndexType::kInternalStorageBitWidth) |
| 308 | return; |
| 309 | // Cluster arguments count clusters per grid. The launch operands specify |
| 310 | // blocks per cluster, so the cluster count is grid size / cluster size. |
| 311 | ConstantIntRanges gridRange = |
| 312 | argRanges[dim].intersection(other: getIndexRange(umin: 1, umax: kMaxDim)); |
| 313 | ConstantIntRanges clusterRange = |
| 314 | argRanges[clusterOperandIndex].intersection( |
| 315 | other: getIndexRange(umin: 1, umax: kMaxClusterDim)); |
| 316 | setRange(intrange::inferDivU(argRanges: {gridRange, clusterRange}), dimResult, |
| 317 | idxResult); |
| 318 | }; |
| 319 | setClusterRange(0, clusterDims.x, clusterIds->x); |
| 320 | setClusterRange(1, clusterDims.y, clusterIds->y); |
| 321 | setClusterRange(2, clusterDims.z, clusterIds->z); |
| 322 | } |
| 323 | } |
| 324 | |