| 1 | //===- GPUTransformOps.cpp - Implementation of GPU transform ops ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.h" |
| 10 | |
| 11 | #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h" |
| 12 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 13 | #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h" |
| 14 | #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h" |
| 15 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 16 | #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h" |
| 17 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 18 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 19 | #include "mlir/Dialect/GPU/TransformOps/Utils.h" |
| 20 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 21 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h" |
| 22 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 23 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 24 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 25 | #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" |
| 26 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 27 | #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
| 28 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
| 29 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 30 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 31 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 32 | #include "mlir/IR/AffineExpr.h" |
| 33 | #include "mlir/IR/Builders.h" |
| 34 | #include "mlir/IR/BuiltinAttributes.h" |
| 35 | #include "mlir/IR/IRMapping.h" |
| 36 | #include "mlir/IR/MLIRContext.h" |
| 37 | #include "mlir/IR/OpDefinition.h" |
| 38 | #include "mlir/IR/Visitors.h" |
| 39 | #include "mlir/Support/LLVM.h" |
| 40 | #include "mlir/Transforms/DialectConversion.h" |
| 41 | #include "llvm/ADT/STLExtras.h" |
| 42 | #include "llvm/ADT/SmallVector.h" |
| 43 | #include "llvm/ADT/TypeSwitch.h" |
| 44 | #include "llvm/Support/DebugLog.h" |
| 45 | #include "llvm/Support/ErrorHandling.h" |
| 46 | #include "llvm/Support/InterleavedRange.h" |
| 47 | #include "llvm/Support/LogicalResult.h" |
| 48 | #include <optional> |
| 49 | #include <type_traits> |
| 50 | |
| 51 | using namespace mlir; |
| 52 | using namespace mlir::gpu; |
| 53 | using namespace mlir::transform; |
| 54 | using namespace mlir::transform::gpu; |
| 55 | |
| 56 | #define DEBUG_TYPE "gpu-transforms" |
| 57 | |
| 58 | //===----------------------------------------------------------------------===// |
| 59 | // Apply...ConversionPatternsOp |
| 60 | //===----------------------------------------------------------------------===// |
| 61 | |
| 62 | void transform::ApplyGPUToNVVMConversionPatternsOp::populatePatterns( |
| 63 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 64 | auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter); |
| 65 | configureGpuToNVVMTypeConverter(converter&: llvmTypeConverter); |
| 66 | // Set higher benefit, so patterns will run before generic LLVM lowering. |
| 67 | populateGpuToNVVMConversionPatterns(converter: llvmTypeConverter, patterns, |
| 68 | benefit: getBenefit()); |
| 69 | } |
| 70 | |
| 71 | LogicalResult |
| 72 | transform::ApplyGPUToNVVMConversionPatternsOp::verifyTypeConverter( |
| 73 | transform::TypeConverterBuilderOpInterface builder) { |
| 74 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 75 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 76 | return success(); |
| 77 | } |
| 78 | |
| 79 | void transform::ApplyGPUWwmaToNVVMConversionPatternsOp::populatePatterns( |
| 80 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 81 | auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter); |
| 82 | populateGpuWMMAToNVVMConversionPatterns(converter: llvmTypeConverter, patterns); |
| 83 | } |
| 84 | |
| 85 | LogicalResult |
| 86 | transform::ApplyGPUWwmaToNVVMConversionPatternsOp::verifyTypeConverter( |
| 87 | transform::TypeConverterBuilderOpInterface builder) { |
| 88 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 89 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 90 | return success(); |
| 91 | } |
| 92 | |
| 93 | void transform::ApplyGPUSubgroupReduceToNVVMConversionPatternsOp:: |
| 94 | populatePatterns(TypeConverter &typeConverter, |
| 95 | RewritePatternSet &patterns) { |
| 96 | auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter); |
| 97 | populateGpuSubgroupReduceOpLoweringPattern(converter: llvmTypeConverter, patterns); |
| 98 | } |
| 99 | |
| 100 | LogicalResult transform::ApplyGPUSubgroupReduceToNVVMConversionPatternsOp:: |
| 101 | verifyTypeConverter(transform::TypeConverterBuilderOpInterface builder) { |
| 102 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 103 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 104 | return success(); |
| 105 | } |
| 106 | |
| 107 | void transform::ApplyGPUToROCDLConversionPatternsOp::populatePatterns( |
| 108 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 109 | auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter); |
| 110 | amdgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: llvmTypeConverter); |
| 111 | // The verifier has already rejected anything unparseable. |
| 112 | FailureOr<ROCDL::TargetInfo> targetInfo = |
| 113 | ROCDL::TargetInfo::get(arch: getArch(), waveSize: getWavesize().value_or(u: 0)); |
| 114 | assert(llvm::succeeded(targetInfo) && "verifier accepted this target" ); |
| 115 | populateGpuToROCDLConversionPatterns( |
| 116 | converter: llvmTypeConverter, patterns, runtime: mlir::gpu::amd::Runtime::HIP, target: *targetInfo); |
| 117 | } |
| 118 | |
| 119 | LogicalResult |
| 120 | transform::ApplyGPUToROCDLConversionPatternsOp::verifyTypeConverter( |
| 121 | transform::TypeConverterBuilderOpInterface builder) { |
| 122 | if (failed(Result: ROCDL::TargetInfo::get(arch: getArch(), waveSize: getWavesize().value_or(u: 0), |
| 123 | emitError: [&] { return emitOpError(); }))) |
| 124 | return failure(); |
| 125 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 126 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 127 | return success(); |
| 128 | } |
| 129 | |
| 130 | //===----------------------------------------------------------------------===// |
| 131 | // Apply...PatternsOp |
| 132 | //===----------------------------------------------------------------------===//s |
| 133 | |
| 134 | void ApplyGPURewritePatternsOp::populatePatterns(RewritePatternSet &patterns) { |
| 135 | populateGpuRewritePatterns(patterns); |
| 136 | } |
| 137 | |
| 138 | void transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::populatePatterns( |
| 139 | RewritePatternSet &patterns) { |
| 140 | std::optional<StringRef> archName = getArch(); |
| 141 | std::optional<ROCDL::TargetInfo> targetInfo; |
| 142 | if (archName) { |
| 143 | // The verifier has already rejected anything unparseable. |
| 144 | FailureOr<ROCDL::TargetInfo> parsed = ROCDL::TargetInfo::get(arch: *archName); |
| 145 | assert(llvm::succeeded(parsed) && "verifier accepted this target" ); |
| 146 | targetInfo = *parsed; |
| 147 | } |
| 148 | |
| 149 | populateGpuPromoteShuffleToAMDGPUPatterns(patterns, target: targetInfo); |
| 150 | } |
| 151 | |
| 152 | LogicalResult transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::verify() { |
| 153 | std::optional<StringRef> archName = getArch(); |
| 154 | if (!archName) |
| 155 | return success(); |
| 156 | |
| 157 | if (failed(Result: ROCDL::TargetInfo::get(arch: *archName, /*waveSize=*/0, |
| 158 | emitError: [&] { return emitOpError(); }))) |
| 159 | return failure(); |
| 160 | return success(); |
| 161 | } |
| 162 | |
| 163 | //===----------------------------------------------------------------------===// |
| 164 | // ApplyUnrollVectorsSubgroupMmaOp |
| 165 | //===----------------------------------------------------------------------===// |
| 166 | |
| 167 | /// Pick an unrolling order that will allow tensorcore operation to reuse LHS |
| 168 | /// register. |
| 169 | static std::optional<SmallVector<int64_t>> |
| 170 | gpuMmaUnrollOrder(vector::ContractionOp contract) { |
| 171 | SmallVector<int64_t> order; |
| 172 | // First make reduction the outer dimensions. |
| 173 | for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) { |
| 174 | if (vector::isReductionIterator(attr: iter)) { |
| 175 | order.push_back(Elt: index); |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | llvm::SmallDenseSet<int64_t> dims; |
| 180 | for (AffineExpr expr : contract.getIndexingMapsArray()[0].getResults()) { |
| 181 | dims.insert(V: cast<AffineDimExpr>(Val&: expr).getPosition()); |
| 182 | } |
| 183 | // Then parallel dimensions that are part of Lhs as we want to re-use Lhs. |
| 184 | for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) { |
| 185 | if (vector::isParallelIterator(attr: iter) && dims.count(V: index)) { |
| 186 | order.push_back(Elt: index); |
| 187 | } |
| 188 | } |
| 189 | // Then the remaining parallel loops. |
| 190 | for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) { |
| 191 | if (vector::isParallelIterator(attr: iter) && !dims.count(V: index)) { |
| 192 | order.push_back(Elt: index); |
| 193 | } |
| 194 | } |
| 195 | return order; |
| 196 | } |
| 197 | |
| 198 | /// Returns the target vector size for the target operation based on the native |
| 199 | /// vector size specified with `m`, `n`, and `k`. |
| 200 | static std::optional<SmallVector<int64_t>> |
| 201 | getSubgroupMmaNativeVectorSize(Operation *op, int64_t m, int64_t n, int64_t k) { |
| 202 | if (auto contract = dyn_cast<vector::ContractionOp>(Val: op)) { |
| 203 | int64_t contractRank = contract.getIteratorTypes().size(); |
| 204 | if (contractRank < 3) |
| 205 | return std::nullopt; |
| 206 | SmallVector<int64_t> nativeSize(contractRank - 3, 1); |
| 207 | nativeSize.append(IL: {m, n, k}); |
| 208 | return nativeSize; |
| 209 | } |
| 210 | if (auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: op)) { |
| 211 | int64_t writeRank = writeOp.getVectorType().getRank(); |
| 212 | if (writeRank < 2) |
| 213 | return std::nullopt; |
| 214 | SmallVector<int64_t> nativeSize(writeRank - 2, 1); |
| 215 | nativeSize.append(IL: {m, n}); |
| 216 | return nativeSize; |
| 217 | } |
| 218 | if (auto readOp = dyn_cast<vector::TransferReadOp>(Val: op)) { |
| 219 | // Transfer read ops may need different shapes based on how they are being |
| 220 | // used. For simplicity just match the shape used by the extract strided op. |
| 221 | VectorType sliceType; |
| 222 | for (Operation *users : op->getUsers()) { |
| 223 | auto = dyn_cast<vector::ExtractStridedSliceOp>(Val: users); |
| 224 | if (!extract) |
| 225 | return std::nullopt; |
| 226 | auto vecType = cast<VectorType>(Val: extract.getResult().getType()); |
| 227 | if (sliceType && sliceType != vecType) |
| 228 | return std::nullopt; |
| 229 | sliceType = vecType; |
| 230 | } |
| 231 | return llvm::to_vector(Range: sliceType.getShape()); |
| 232 | } |
| 233 | if ((OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1)) { |
| 234 | if (auto vecType = dyn_cast<VectorType>(Val: op->getResultTypes()[0])) { |
| 235 | // TODO: The condition for unrolling elementwise should be restricted |
| 236 | // only to operations that need unrolling (connected to the contract). |
| 237 | if (vecType.getRank() < 2) |
| 238 | return std::nullopt; |
| 239 | |
| 240 | // First check whether there is a slice to infer the shape from. This is |
| 241 | // required for cases where the accumulator type differs from the input |
| 242 | // types, in which case we will see an `arith.ext_` between the contract |
| 243 | // and transfer_read which needs to be unrolled. |
| 244 | VectorType sliceType; |
| 245 | for (Operation *users : op->getUsers()) { |
| 246 | auto = dyn_cast<vector::ExtractStridedSliceOp>(Val: users); |
| 247 | if (!extract) |
| 248 | return std::nullopt; |
| 249 | auto vecType = cast<VectorType>(Val: extract.getResult().getType()); |
| 250 | if (sliceType && sliceType != vecType) |
| 251 | return std::nullopt; |
| 252 | sliceType = vecType; |
| 253 | } |
| 254 | if (sliceType) |
| 255 | return llvm::to_vector(Range: sliceType.getShape()); |
| 256 | |
| 257 | // Else unroll for trailing elementwise. |
| 258 | SmallVector<int64_t> nativeSize(vecType.getRank() - 2, 1); |
| 259 | // Map elementwise ops to the output shape. |
| 260 | nativeSize.append(IL: {m, n}); |
| 261 | return nativeSize; |
| 262 | } |
| 263 | } |
| 264 | return std::nullopt; |
| 265 | } |
| 266 | |
| 267 | void transform::ApplyUnrollVectorsSubgroupMmaOp::( |
| 268 | RewritePatternSet &patterns) { |
| 269 | auto unrollOrder = [](Operation *op) -> std::optional<SmallVector<int64_t>> { |
| 270 | auto contract = dyn_cast<vector::ContractionOp>(Val: op); |
| 271 | if (!contract) |
| 272 | return std::nullopt; |
| 273 | return gpuMmaUnrollOrder(contract); |
| 274 | }; |
| 275 | |
| 276 | int64_t m = getM(); |
| 277 | int64_t n = getN(); |
| 278 | int64_t k = getK(); |
| 279 | auto nativeShapeFn = |
| 280 | [m, n, k](Operation *op) -> std::optional<SmallVector<int64_t>> { |
| 281 | return getSubgroupMmaNativeVectorSize(op, m, n, k); |
| 282 | }; |
| 283 | vector::populateVectorUnrollPatterns( |
| 284 | patterns, options: vector::UnrollVectorOptions() |
| 285 | .setNativeShapeFn(nativeShapeFn) |
| 286 | .setUnrollTraversalOrderFn(unrollOrder)); |
| 287 | } |
| 288 | |
| 289 | //===----------------------------------------------------------------------===// |
| 290 | // EliminateBarriersOp |
| 291 | //===----------------------------------------------------------------------===// |
| 292 | |
| 293 | void EliminateBarriersOp::populatePatterns(RewritePatternSet &patterns) { |
| 294 | populateGpuEliminateBarriersPatterns(patterns); |
| 295 | } |
| 296 | |
| 297 | //===----------------------------------------------------------------------===// |
| 298 | // Block and thread mapping utilities. |
| 299 | //===----------------------------------------------------------------------===// |
| 300 | |
| 301 | namespace { |
| 302 | /// Local types used for mapping verification. |
| 303 | struct MappingKind {}; |
| 304 | struct BlockMappingKind : MappingKind {}; |
| 305 | struct ThreadMappingKind : MappingKind {}; |
| 306 | } // namespace |
| 307 | |
| 308 | static DiagnosedSilenceableFailure |
| 309 | definiteFailureHelper(std::optional<TransformOpInterface> transformOp, |
| 310 | Operation *target, const Twine &message) { |
| 311 | if (transformOp.has_value()) |
| 312 | return transformOp->emitDefiniteFailure() << message; |
| 313 | return emitDefiniteFailure(op: target, message); |
| 314 | } |
| 315 | |
| 316 | /// Check if given mapping attributes are one of the desired attributes |
| 317 | template <typename MappingKindType> |
| 318 | static DiagnosedSilenceableFailure |
| 319 | checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp, |
| 320 | scf::ForallOp forallOp) { |
| 321 | if (!forallOp.getMapping().has_value()) { |
| 322 | return definiteFailureHelper(transformOp, target: forallOp, |
| 323 | message: "scf.forall op requires a mapping attribute" ); |
| 324 | } |
| 325 | |
| 326 | bool hasBlockMapping = llvm::any_of(Range: forallOp.getMapping().value(), |
| 327 | P: llvm::IsaPred<GPUBlockMappingAttr>); |
| 328 | bool hasWarpgroupMapping = llvm::any_of( |
| 329 | Range: forallOp.getMapping().value(), P: llvm::IsaPred<GPUWarpgroupMappingAttr>); |
| 330 | bool hasWarpMapping = llvm::any_of(Range: forallOp.getMapping().value(), |
| 331 | P: llvm::IsaPred<GPUWarpMappingAttr>); |
| 332 | bool hasThreadMapping = llvm::any_of(Range: forallOp.getMapping().value(), |
| 333 | P: llvm::IsaPred<GPUThreadMappingAttr>); |
| 334 | bool hasLaneMapping = llvm::any_of(Range: forallOp.getMapping().value(), |
| 335 | P: llvm::IsaPred<GPULaneMappingAttr>); |
| 336 | int64_t countMappingTypes = 0; |
| 337 | countMappingTypes += hasBlockMapping ? 1 : 0; |
| 338 | countMappingTypes += hasWarpgroupMapping ? 1 : 0; |
| 339 | countMappingTypes += hasWarpMapping ? 1 : 0; |
| 340 | countMappingTypes += hasThreadMapping ? 1 : 0; |
| 341 | countMappingTypes += hasLaneMapping ? 1 : 0; |
| 342 | if (countMappingTypes > 1) { |
| 343 | return definiteFailureHelper( |
| 344 | transformOp, target: forallOp, |
| 345 | message: "cannot mix different mapping types, use nesting" ); |
| 346 | } |
| 347 | if (std::is_same<MappingKindType, BlockMappingKind>::value && |
| 348 | !hasBlockMapping) { |
| 349 | return definiteFailureHelper( |
| 350 | transformOp, target: forallOp, |
| 351 | message: "scf.forall op requires a mapping attribute of kind 'block'" ); |
| 352 | } |
| 353 | if (std::is_same<MappingKindType, ThreadMappingKind>::value && |
| 354 | !hasLaneMapping && !hasThreadMapping && !hasWarpMapping && |
| 355 | !hasWarpgroupMapping) { |
| 356 | return definiteFailureHelper(transformOp, target: forallOp, |
| 357 | message: "scf.forall op requires a mapping attribute " |
| 358 | "of kind 'thread' or 'warp'" ); |
| 359 | } |
| 360 | |
| 361 | DenseSet<Attribute> seen; |
| 362 | for (Attribute map : forallOp.getMapping()->getValue()) { |
| 363 | if (seen.contains(V: map)) { |
| 364 | return definiteFailureHelper( |
| 365 | transformOp, target: forallOp, |
| 366 | message: "duplicate attribute, cannot map different loops " |
| 367 | "to the same mapping id" ); |
| 368 | } |
| 369 | seen.insert(V: map); |
| 370 | } |
| 371 | |
| 372 | auto isLinear = [](DeviceMappingAttrInterface attr) { |
| 373 | return attr.isLinearMapping(); |
| 374 | }; |
| 375 | if (llvm::any_of(forallOp.getDeviceMappingAttrs(), isLinear) && |
| 376 | !llvm::all_of(forallOp.getDeviceMappingAttrs(), isLinear)) { |
| 377 | return definiteFailureHelper( |
| 378 | transformOp, target: forallOp, |
| 379 | message: "cannot mix linear and non-linear mapping modes" ); |
| 380 | } |
| 381 | |
| 382 | FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr = |
| 383 | forallOp.getDeviceMaskingAttr(); |
| 384 | if (succeeded(Result: maybeMaskingAttr) && *maybeMaskingAttr && |
| 385 | !forallOp.usesLinearMapping()) { |
| 386 | return definiteFailureHelper( |
| 387 | transformOp, target: forallOp, |
| 388 | message: "device masking is only available in linear mapping mode" ); |
| 389 | } |
| 390 | |
| 391 | return DiagnosedSilenceableFailure::success(); |
| 392 | } |
| 393 | |
| 394 | template <typename MappingKindType> |
| 395 | static DiagnosedSilenceableFailure |
| 396 | verifyGpuMapping(std::optional<TransformOpInterface> transformOp, |
| 397 | scf::ForallOp forallOp) { |
| 398 | // Check the types of the mapping attributes match. |
| 399 | DiagnosedSilenceableFailure typeRes = |
| 400 | checkMappingAttributeTypes<MappingKindType>(transformOp, forallOp); |
| 401 | if (!typeRes.succeeded()) |
| 402 | return typeRes; |
| 403 | |
| 404 | // Perform other non-types verifications. |
| 405 | if (!forallOp.isNormalized()) |
| 406 | return definiteFailureHelper(transformOp, target: forallOp, |
| 407 | message: "unsupported non-normalized loops" ); |
| 408 | if (forallOp.getNumResults() > 0) |
| 409 | return definiteFailureHelper(transformOp, target: forallOp, |
| 410 | message: "only bufferized scf.forall can be mapped" ); |
| 411 | bool useLinearMapping = forallOp.usesLinearMapping(); |
| 412 | // TODO: This would be more natural with support for Optional<EnumParameter> |
| 413 | // in GPUDeviceMappingAttr. |
| 414 | int64_t maxNumMappingsSupported = |
| 415 | useLinearMapping ? (getMaxEnumValForMappingId() - |
| 416 | static_cast<uint64_t>(MappingId::DimZ)) |
| 417 | : 3; |
| 418 | if (forallOp.getRank() > maxNumMappingsSupported) { |
| 419 | return definiteFailureHelper(transformOp, target: forallOp, |
| 420 | message: "scf.forall with rank > " ) |
| 421 | << maxNumMappingsSupported |
| 422 | << " does not lower for the specified mapping attribute type" ; |
| 423 | } |
| 424 | auto numParallelIterations = |
| 425 | getConstantIntValues(ofrs: forallOp.getMixedUpperBound()); |
| 426 | if (!forallOp.isNormalized() || !numParallelIterations.has_value()) { |
| 427 | return definiteFailureHelper( |
| 428 | transformOp, target: forallOp, |
| 429 | message: "requires statically sized, normalized forall op" ); |
| 430 | } |
| 431 | return DiagnosedSilenceableFailure::success(); |
| 432 | } |
| 433 | |
| 434 | /// Struct to return the result of the rewrite of a forall operation. |
| 435 | struct ForallRewriteResult { |
| 436 | SmallVector<int64_t> mappingSizes; |
| 437 | SmallVector<Value> mappingIds; |
| 438 | }; |
| 439 | |
| 440 | /// Helper to replace ids of dimensions known to be 1 by 0 to simplify the IR. |
| 441 | template <typename OpTy, typename OperationOrBlock> |
| 442 | static void |
| 443 | replaceUnitMappingIdsHelper(RewriterBase &rewriter, Location loc, |
| 444 | OperationOrBlock *parent, Value replacement, |
| 445 | ArrayRef<int64_t> availableMappingSizes) { |
| 446 | parent->walk([&](OpTy idOp) { |
| 447 | if (availableMappingSizes[static_cast<int64_t>(idOp.getDimension())] == 1) |
| 448 | rewriter.replaceAllUsesWith(idOp.getResult(), replacement); |
| 449 | }); |
| 450 | } |
| 451 | |
| 452 | static DiagnosedSilenceableFailure rewriteOneForallCommonImpl( |
| 453 | RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp, |
| 454 | scf::ForallOp forallOp, ArrayRef<int64_t> availableMappingSizes, |
| 455 | ForallRewriteResult &result, const GpuIdBuilder &gpuIdBuilder) { |
| 456 | LDBG() << "--start rewriteOneForallCommonImpl" ; |
| 457 | |
| 458 | // Step 1. Complete the mapping to a full mapping (with 1s) if necessary. |
| 459 | auto numParallelIterations = |
| 460 | getConstantIntValues(ofrs: forallOp.getMixedUpperBound()); |
| 461 | assert(forallOp.isNormalized() && numParallelIterations.has_value() && |
| 462 | "requires statically sized, normalized forall op" ); |
| 463 | SmallVector<int64_t> tmpMappingSizes = numParallelIterations.value(); |
| 464 | SmallVector<DeviceMappingAttrInterface> forallMappingAttrsVec = |
| 465 | forallOp.getDeviceMappingAttrs(); |
| 466 | SetVector<Attribute> forallMappingAttrs; |
| 467 | forallMappingAttrs.insert_range(R&: forallMappingAttrsVec); |
| 468 | auto comparator = [](Attribute a, Attribute b) -> bool { |
| 469 | return cast<DeviceMappingAttrInterface>(Val&: a).getMappingId() < |
| 470 | cast<DeviceMappingAttrInterface>(Val&: b).getMappingId(); |
| 471 | }; |
| 472 | |
| 473 | // Step 1.b. In the linear case, compute the max mapping to avoid needlessly |
| 474 | // mapping all dimensions. In the 3-D mapping case we need to map all |
| 475 | // dimensions. |
| 476 | DeviceMappingAttrInterface maxMapping = cast<DeviceMappingAttrInterface>( |
| 477 | Val: *llvm::max_element(Range&: forallMappingAttrs, C: comparator)); |
| 478 | DeviceMappingAttrInterface maxLinearMapping; |
| 479 | if (maxMapping.isLinearMapping()) |
| 480 | maxLinearMapping = maxMapping; |
| 481 | for (auto attr : gpuIdBuilder.mappingAttributes) { |
| 482 | // If attr overflows, just skip. |
| 483 | if (maxLinearMapping && comparator(maxLinearMapping, attr)) |
| 484 | continue; |
| 485 | // Try to insert. If element was already present, just continue. |
| 486 | if (!forallMappingAttrs.insert(X: attr)) |
| 487 | continue; |
| 488 | // Otherwise, we have a new insertion without a size -> use size 1. |
| 489 | tmpMappingSizes.push_back(Elt: 1); |
| 490 | } |
| 491 | LDBG() << "----tmpMappingSizes extracted from scf.forall op: " |
| 492 | << llvm::interleaved(R: tmpMappingSizes); |
| 493 | |
| 494 | // Step 2. sort the values by the corresponding DeviceMappingAttrInterface. |
| 495 | SmallVector<int64_t> forallMappingSizes = getValuesSortedByKey( |
| 496 | keys: forallMappingAttrs.getArrayRef(), values: tmpMappingSizes, compare: comparator); |
| 497 | LDBG() << "----forallMappingSizes: " << llvm::interleaved(R: forallMappingSizes); |
| 498 | LDBG() << "----forallMappingAttrs: " << llvm::interleaved(R: forallMappingAttrs); |
| 499 | |
| 500 | // Step 3. Generate the mappingIdOps using the provided generator. |
| 501 | Location loc = forallOp.getLoc(); |
| 502 | OpBuilder::InsertionGuard guard(rewriter); |
| 503 | rewriter.setInsertionPoint(forallOp); |
| 504 | SmallVector<int64_t> originalBasis(availableMappingSizes); |
| 505 | bool originalBasisWasProvided = !originalBasis.empty(); |
| 506 | if (!originalBasisWasProvided) { |
| 507 | LDBG() << "----originalBasis was not provided, deriving it and there will " |
| 508 | "be no " |
| 509 | "predication" ; |
| 510 | originalBasis = forallMappingSizes; |
| 511 | while (originalBasis.size() < 3) |
| 512 | originalBasis.push_back(Elt: 1); |
| 513 | } else { |
| 514 | LDBG() << "----originalBasis was provided, using it, there will be " |
| 515 | "predication" ; |
| 516 | } |
| 517 | LDBG() << "------originalBasis: " << llvm::interleaved(R: originalBasis); |
| 518 | |
| 519 | IdBuilderResult builderResult = |
| 520 | gpuIdBuilder.idBuilder(rewriter, loc, forallMappingSizes, originalBasis); |
| 521 | if (!builderResult.errorMsg.empty()) |
| 522 | return definiteFailureHelper(transformOp, target: forallOp, message: builderResult.errorMsg); |
| 523 | |
| 524 | LDBG() << builderResult; |
| 525 | |
| 526 | // Step 4. Map the induction variables to the mappingIdOps, this may involve |
| 527 | // a permutation. |
| 528 | SmallVector<Value> mappingIdOps = builderResult.mappingIdOps; |
| 529 | IRMapping bvm; |
| 530 | for (auto [iv, dim] : llvm::zip_equal( |
| 531 | t: forallOp.getInductionVars(), |
| 532 | u: forallMappingAttrs.getArrayRef().take_front(N: forallOp.getRank()))) { |
| 533 | auto mappingAttr = cast<DeviceMappingAttrInterface>(Val: dim); |
| 534 | Value peIdOp = mappingIdOps[mappingAttr.getRelativeIndex()]; |
| 535 | LDBG() << "----map: " << iv << " to " << peIdOp; |
| 536 | bvm.map(from: iv, to: peIdOp); |
| 537 | } |
| 538 | |
| 539 | // Step 5. If the originalBasis is already known, create conditionals to |
| 540 | // predicate the region. Otherwise, the current forall determines the |
| 541 | // originalBasis and no predication occurs. |
| 542 | Value predicate; |
| 543 | if (originalBasisWasProvided) { |
| 544 | for (Value tmpPredicate : builderResult.predicateOps) { |
| 545 | predicate = predicate ? arith::AndIOp::create(builder&: rewriter, location: loc, lhs: predicate, |
| 546 | rhs: tmpPredicate) |
| 547 | : tmpPredicate; |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | // Step 6. Move the body of forallOp. |
| 552 | // Erase the terminator first, it will not be used. |
| 553 | rewriter.eraseOp(op: forallOp.getTerminator()); |
| 554 | Block *targetBlock; |
| 555 | Block::iterator insertionPoint; |
| 556 | if (predicate) { |
| 557 | // Step 6.a. If predicated, move at the beginning. |
| 558 | auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, |
| 559 | /*withElseRegion=*/false); |
| 560 | targetBlock = ifOp.thenBlock(); |
| 561 | insertionPoint = ifOp.thenBlock()->begin(); |
| 562 | } else { |
| 563 | // Step 6.b. Otherwise, move inline just at the rewriter insertion |
| 564 | // point. |
| 565 | targetBlock = forallOp->getBlock(); |
| 566 | insertionPoint = rewriter.getInsertionPoint(); |
| 567 | } |
| 568 | Block &sourceBlock = forallOp.getRegion().front(); |
| 569 | targetBlock->getOperations().splice(where: insertionPoint, |
| 570 | L2&: sourceBlock.getOperations()); |
| 571 | |
| 572 | // Step 7. RAUW indices. |
| 573 | for (Value loopIndex : forallOp.getInductionVars()) { |
| 574 | Value threadIdx = bvm.lookup(from: loopIndex); |
| 575 | rewriter.replaceAllUsesWith(from: loopIndex, to: threadIdx); |
| 576 | } |
| 577 | |
| 578 | // Step 8. Erase old op. |
| 579 | rewriter.eraseOp(op: forallOp); |
| 580 | |
| 581 | LDBG() << "----result forallMappingSizes: " |
| 582 | << llvm::interleaved(R: forallMappingSizes); |
| 583 | LDBG() << "----result mappingIdOps: " << llvm::interleaved(R: mappingIdOps); |
| 584 | |
| 585 | result = ForallRewriteResult{.mappingSizes: forallMappingSizes, .mappingIds: mappingIdOps}; |
| 586 | return DiagnosedSilenceableFailure::success(); |
| 587 | } |
| 588 | |
| 589 | //===----------------------------------------------------------------------===// |
| 590 | // MapForallToBlocks |
| 591 | //===----------------------------------------------------------------------===// |
| 592 | |
| 593 | DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl( |
| 594 | RewriterBase &rewriter, TransformOpInterface transformOp, |
| 595 | scf::ForallOp forallOp, SmallVectorImpl<int64_t> &gridDims, |
| 596 | const GpuIdBuilder &gpuIdBuilder) { |
| 597 | LDBG() << "Start mapForallToBlocksImpl" ; |
| 598 | |
| 599 | { |
| 600 | // GPU-specific verifications. There is no better place to anchor |
| 601 | // those right now: the ForallOp is target-independent and the transform |
| 602 | // op does not apply to individual ForallOp. |
| 603 | DiagnosedSilenceableFailure diag = |
| 604 | verifyGpuMapping<BlockMappingKind>(transformOp, forallOp); |
| 605 | if (!diag.succeeded()) |
| 606 | return diag; |
| 607 | } |
| 608 | |
| 609 | Location loc = forallOp.getLoc(); |
| 610 | Block *parentBlock = forallOp->getBlock(); |
| 611 | Value zero; |
| 612 | { |
| 613 | // Create an early zero index value for replacements and immediately reset |
| 614 | // the insertion point. |
| 615 | OpBuilder::InsertionGuard guard(rewriter); |
| 616 | rewriter.setInsertionPointToStart(parentBlock); |
| 617 | zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 618 | } |
| 619 | |
| 620 | ForallRewriteResult rewriteResult; |
| 621 | DiagnosedSilenceableFailure diag = rewriteOneForallCommonImpl( |
| 622 | rewriter, transformOp, forallOp, |
| 623 | /*availableMappingSizes=*/gridDims, result&: rewriteResult, gpuIdBuilder); |
| 624 | |
| 625 | // Return if anything goes wrong, use silenceable failure as a match |
| 626 | // failure. |
| 627 | if (!diag.succeeded()) |
| 628 | return diag; |
| 629 | |
| 630 | // If gridDims was not provided already, set it from the return. |
| 631 | if (gridDims.empty()) { |
| 632 | gridDims = rewriteResult.mappingSizes; |
| 633 | while (gridDims.size() < 3) |
| 634 | gridDims.push_back(Elt: 1); |
| 635 | } |
| 636 | assert(gridDims.size() == 3 && "Need 3-D gridDims" ); |
| 637 | |
| 638 | // Replace ids of dimensions known to be 1 by 0 to simplify the IR. |
| 639 | // Here, the result of mapping determines the available mapping sizes. |
| 640 | replaceUnitMappingIdsHelper<BlockDimOp>(rewriter, loc, parent: parentBlock, replacement: zero, |
| 641 | availableMappingSizes: rewriteResult.mappingSizes); |
| 642 | |
| 643 | return DiagnosedSilenceableFailure::success(); |
| 644 | } |
| 645 | |
| 646 | DiagnosedSilenceableFailure |
| 647 | mlir::transform::gpu::findTopLevelForallOp(Operation *target, |
| 648 | scf::ForallOp &topLevelForallOp, |
| 649 | TransformOpInterface transformOp) { |
| 650 | auto walkResult = target->walk(callback: [&](scf::ForallOp forallOp) { |
| 651 | if (forallOp->getParentOfType<scf::ForallOp>()) |
| 652 | return WalkResult::advance(); |
| 653 | if (topLevelForallOp) |
| 654 | // TODO: Handle multiple forall if they are independent. |
| 655 | return WalkResult::interrupt(); |
| 656 | topLevelForallOp = forallOp; |
| 657 | return WalkResult::advance(); |
| 658 | }); |
| 659 | |
| 660 | if (walkResult.wasInterrupted() || !topLevelForallOp) |
| 661 | return transformOp.emitSilenceableError() |
| 662 | << "could not find a unique topLevel scf.forall" ; |
| 663 | return DiagnosedSilenceableFailure::success(); |
| 664 | } |
| 665 | |
| 666 | DiagnosedSilenceableFailure transform::MapForallToBlocks::applyToOne( |
| 667 | transform::TransformRewriter &rewriter, Operation *target, |
| 668 | ApplyToEachResultList &results, transform::TransformState &state) { |
| 669 | LaunchOp gpuLaunch = dyn_cast<LaunchOp>(Val: target); |
| 670 | auto transformOp = cast<TransformOpInterface>(Val: getOperation()); |
| 671 | |
| 672 | if (!getGenerateGpuLaunch() && !gpuLaunch) { |
| 673 | DiagnosedSilenceableFailure diag = |
| 674 | emitSilenceableError() |
| 675 | << "Given target is not gpu.launch, set `generate_gpu_launch` " |
| 676 | "attribute" ; |
| 677 | diag.attachNote(loc: target->getLoc()) << "when applied to this payload op" ; |
| 678 | return diag; |
| 679 | } |
| 680 | |
| 681 | scf::ForallOp topLevelForallOp; |
| 682 | DiagnosedSilenceableFailure diag = mlir::transform::gpu::findTopLevelForallOp( |
| 683 | target, topLevelForallOp, transformOp); |
| 684 | if (!diag.succeeded()) { |
| 685 | diag.attachNote(loc: target->getLoc()) << "when applied to this payload op" ; |
| 686 | return diag; |
| 687 | } |
| 688 | assert(topLevelForallOp && "expect an scf.forall" ); |
| 689 | |
| 690 | SmallVector<int64_t> gridDims{getGridDims()}; |
| 691 | if (!getGenerateGpuLaunch() && gridDims.size() != 3) |
| 692 | return transformOp.emitDefiniteFailure(message: "transform require size-3 mapping" ); |
| 693 | |
| 694 | OpBuilder::InsertionGuard guard(rewriter); |
| 695 | rewriter.setInsertionPoint(topLevelForallOp); |
| 696 | |
| 697 | // Generate gpu launch here and move the forall inside |
| 698 | if (getGenerateGpuLaunch()) { |
| 699 | DiagnosedSilenceableFailure diag = |
| 700 | createGpuLaunch(rewriter, loc: target->getLoc(), transformOp, launchOp&: gpuLaunch); |
| 701 | if (!diag.succeeded()) |
| 702 | return diag; |
| 703 | |
| 704 | rewriter.setInsertionPointToStart(&gpuLaunch.getBody().front()); |
| 705 | Operation *newForallOp = rewriter.clone(op&: *topLevelForallOp); |
| 706 | rewriter.eraseOp(op: topLevelForallOp); |
| 707 | topLevelForallOp = cast<scf::ForallOp>(Val: newForallOp); |
| 708 | } |
| 709 | |
| 710 | // The BlockIdBuilder adapts to whatever is thrown at it. |
| 711 | bool useLinearMapping = false; |
| 712 | if (topLevelForallOp.getMapping()) |
| 713 | useLinearMapping = topLevelForallOp.usesLinearMapping(); |
| 714 | |
| 715 | FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr = |
| 716 | topLevelForallOp.getDeviceMaskingAttr(); |
| 717 | assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr" ); |
| 718 | assert((!*maybeMaskingAttr || useLinearMapping) && |
| 719 | "masking requires linear mapping" ); |
| 720 | |
| 721 | GpuBlockIdBuilder gpuBlockIdBuilder(getContext(), useLinearMapping, |
| 722 | *maybeMaskingAttr); |
| 723 | |
| 724 | diag = mlir::transform::gpu::mapForallToBlocksImpl( |
| 725 | rewriter, transformOp, forallOp: topLevelForallOp, gridDims, gpuIdBuilder: gpuBlockIdBuilder); |
| 726 | if (!diag.succeeded()) |
| 727 | return diag; |
| 728 | |
| 729 | // Set the GPU launch configuration for the grid dims late, this is |
| 730 | // subject to IR inspection. |
| 731 | diag = alterGpuLaunch(rewriter, gpuLaunch, |
| 732 | transformOp: cast<TransformOpInterface>(Val: getOperation()), gridDimX: gridDims[0], |
| 733 | gridDimY: gridDims[1], gridDimZ: gridDims[2]); |
| 734 | |
| 735 | results.push_back(op: gpuLaunch); |
| 736 | return diag; |
| 737 | } |
| 738 | |
| 739 | LogicalResult transform::MapForallToBlocks::verify() { |
| 740 | if (!getGridDims().empty() && getGridDims().size() != 3) { |
| 741 | return emitOpError() << "transform requires empty or size-3 grid_dims" ; |
| 742 | } |
| 743 | return success(); |
| 744 | } |
| 745 | |
| 746 | //===----------------------------------------------------------------------===// |
| 747 | // MapNestedForallToThreads |
| 748 | //===----------------------------------------------------------------------===// |
| 749 | |
| 750 | static DiagnosedSilenceableFailure checkMappingSpec( |
| 751 | std::optional<TransformOpInterface> transformOp, scf::ForallOp forallOp, |
| 752 | ArrayRef<int64_t> numParallelIterations, ArrayRef<int64_t> blockOrGridSizes, |
| 753 | int factor, bool useLinearMapping = false) { |
| 754 | if (llvm::any_of(Range&: blockOrGridSizes, P: [](int64_t i) { return i <= 0; })) { |
| 755 | return definiteFailureHelper(transformOp, target: forallOp, |
| 756 | message: "block/grid sizes must be strictly positive" ); |
| 757 | } |
| 758 | if (!useLinearMapping && blockOrGridSizes.front() % factor != 0) { |
| 759 | auto diag = definiteFailureHelper( |
| 760 | transformOp, target: forallOp, |
| 761 | message: Twine("3-D mapping: size of threadIdx.x must be a multiple of " ) + |
| 762 | Twine(factor)); |
| 763 | return diag; |
| 764 | } |
| 765 | bool hasZeroParallelIteration = |
| 766 | llvm::any_of(Range&: numParallelIterations, P: [](int64_t i) { return i == 0; }); |
| 767 | // `computeProduct` requires strictly positive inputs, so handle the |
| 768 | // zero-iteration case explicitly to avoid asserting on valid degenerate |
| 769 | // loop bounds. |
| 770 | int64_t requiredResourceCount = |
| 771 | hasZeroParallelIteration ? 0 |
| 772 | : computeProduct(basis: numParallelIterations) * factor; |
| 773 | int64_t availableResourceCount = computeProduct(basis: blockOrGridSizes); |
| 774 | if (requiredResourceCount > availableResourceCount) { |
| 775 | auto diag = definiteFailureHelper( |
| 776 | transformOp, target: forallOp, |
| 777 | message: Twine("the number of required parallel resources (blocks or " |
| 778 | "threads) " ) + |
| 779 | Twine(requiredResourceCount) + |
| 780 | " overflows the number of available resources " + |
| 781 | Twine(availableResourceCount)); |
| 782 | return diag; |
| 783 | } |
| 784 | return DiagnosedSilenceableFailure::success(); |
| 785 | } |
| 786 | |
| 787 | static DiagnosedSilenceableFailure |
| 788 | getThreadIdBuilder(std::optional<TransformOpInterface> transformOp, |
| 789 | scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes, |
| 790 | int64_t warpSize, GpuIdBuilder &gpuIdBuilder) { |
| 791 | DeviceMappingAttrInterface mappingAttr = |
| 792 | forallOp.getDeviceMappingAttrs().front(); |
| 793 | bool useLinearMapping = mappingAttr.isLinearMapping(); |
| 794 | |
| 795 | // Sanity checks that may result in runtime verification errors. |
| 796 | auto numParallelIterations = |
| 797 | getConstantIntValues(ofrs: (forallOp.getMixedUpperBound())); |
| 798 | if (!forallOp.isNormalized() || !numParallelIterations.has_value()) { |
| 799 | return definiteFailureHelper( |
| 800 | transformOp, target: forallOp, |
| 801 | message: "requires statically sized, normalized forall op" ); |
| 802 | } |
| 803 | int64_t factor = 1; |
| 804 | if (isa<GPUWarpgroupMappingAttr>(Val: mappingAttr)) { |
| 805 | factor = GpuWarpgroupIdBuilder::kNumWarpsPerGroup * warpSize; |
| 806 | } else if (isa<GPUWarpMappingAttr>(Val: mappingAttr)) { |
| 807 | factor = warpSize; |
| 808 | } |
| 809 | DiagnosedSilenceableFailure diag = |
| 810 | checkMappingSpec(transformOp, forallOp, numParallelIterations: numParallelIterations.value(), |
| 811 | blockOrGridSizes: blockSizes, factor, useLinearMapping); |
| 812 | if (!diag.succeeded()) |
| 813 | return diag; |
| 814 | |
| 815 | FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr = |
| 816 | forallOp.getDeviceMaskingAttr(); |
| 817 | assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr" ); |
| 818 | assert((!*maybeMaskingAttr || useLinearMapping) && |
| 819 | "masking requires linear mapping" ); |
| 820 | |
| 821 | // Start mapping. |
| 822 | MLIRContext *ctx = forallOp.getContext(); |
| 823 | gpuIdBuilder = |
| 824 | TypeSwitch<DeviceMappingAttrInterface, GpuIdBuilder>(mappingAttr) |
| 825 | .Case(caseFn: [&](GPUWarpgroupMappingAttr) { |
| 826 | return GpuWarpgroupIdBuilder(ctx, warpSize, useLinearMapping, |
| 827 | *maybeMaskingAttr); |
| 828 | }) |
| 829 | .Case(caseFn: [&](GPUWarpMappingAttr) { |
| 830 | return GpuWarpIdBuilder(ctx, warpSize, useLinearMapping, |
| 831 | *maybeMaskingAttr); |
| 832 | }) |
| 833 | .Case(caseFn: [&](GPUThreadMappingAttr) { |
| 834 | return GpuThreadIdBuilder(ctx, useLinearMapping, *maybeMaskingAttr); |
| 835 | }) |
| 836 | .Case(caseFn: [&](GPULaneMappingAttr) { |
| 837 | return GpuLaneIdBuilder(ctx, warpSize, useLinearMapping, |
| 838 | *maybeMaskingAttr); |
| 839 | }) |
| 840 | .DefaultUnreachable(message: "unknown mapping attribute" ); |
| 841 | return DiagnosedSilenceableFailure::success(); |
| 842 | } |
| 843 | |
| 844 | DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl( |
| 845 | RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp, |
| 846 | scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes, int64_t warpSize, |
| 847 | bool syncAfterDistribute) { |
| 848 | |
| 849 | { |
| 850 | // GPU-specific verifications. There is no better place to anchor |
| 851 | // those right now: the ForallOp is target-independent and the transform |
| 852 | // op does not apply to individual ForallOp. |
| 853 | DiagnosedSilenceableFailure diag = |
| 854 | verifyGpuMapping<ThreadMappingKind>(transformOp, forallOp); |
| 855 | if (!diag.succeeded()) |
| 856 | return diag; |
| 857 | } |
| 858 | |
| 859 | GpuIdBuilder gpuIdBuilder; |
| 860 | { |
| 861 | // Try to construct the id builder, if it fails, return. |
| 862 | DiagnosedSilenceableFailure diag = getThreadIdBuilder( |
| 863 | transformOp, forallOp, blockSizes, warpSize, gpuIdBuilder); |
| 864 | if (!diag.succeeded()) |
| 865 | return diag; |
| 866 | } |
| 867 | |
| 868 | Location loc = forallOp.getLoc(); |
| 869 | OpBuilder::InsertionGuard g(rewriter); |
| 870 | // Insert after to allow for syncthreads after `forall` is erased. |
| 871 | rewriter.setInsertionPointAfter(forallOp); |
| 872 | ForallRewriteResult rewriteResult; |
| 873 | DiagnosedSilenceableFailure diag = rewriteOneForallCommonImpl( |
| 874 | rewriter, transformOp, forallOp, availableMappingSizes: blockSizes, result&: rewriteResult, gpuIdBuilder); |
| 875 | if (!diag.succeeded()) |
| 876 | return diag; |
| 877 | // Add a syncthreads if needed. TODO: warpsync |
| 878 | if (syncAfterDistribute) |
| 879 | BarrierOp::create(builder&: rewriter, location: loc); |
| 880 | |
| 881 | return DiagnosedSilenceableFailure::success(); |
| 882 | } |
| 883 | |
| 884 | DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl( |
| 885 | RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp, |
| 886 | Operation *target, ArrayRef<int64_t> blockDims, int64_t warpSize, |
| 887 | bool syncAfterDistribute) { |
| 888 | LDBG() << "Start mapNestedForallToThreadsImpl" ; |
| 889 | if (blockDims.size() != 3) { |
| 890 | return definiteFailureHelper(transformOp, target, |
| 891 | message: "requires size-3 thread mapping" ); |
| 892 | } |
| 893 | |
| 894 | // Create an early zero index value for replacements. |
| 895 | Location loc = target->getLoc(); |
| 896 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 897 | DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success(); |
| 898 | WalkResult walkResult = target->walk(callback: [&](scf::ForallOp forallOp) { |
| 899 | diag = mlir::transform::gpu::mapOneForallToThreadsImpl( |
| 900 | rewriter, transformOp, forallOp, blockSizes: blockDims, warpSize, |
| 901 | syncAfterDistribute); |
| 902 | if (diag.isDefiniteFailure()) |
| 903 | return WalkResult::interrupt(); |
| 904 | if (diag.succeeded()) |
| 905 | return WalkResult::skip(); |
| 906 | return WalkResult::advance(); |
| 907 | }); |
| 908 | if (walkResult.wasInterrupted()) |
| 909 | return diag; |
| 910 | |
| 911 | // Replace ids of dimensions known to be 1 by 0 to simplify the IR. |
| 912 | // Here, the result of mapping determines the available mapping sizes. |
| 913 | replaceUnitMappingIdsHelper<ThreadIdOp>(rewriter, loc, parent: target, replacement: zero, |
| 914 | availableMappingSizes: blockDims); |
| 915 | |
| 916 | return DiagnosedSilenceableFailure::success(); |
| 917 | } |
| 918 | |
| 919 | DiagnosedSilenceableFailure transform::MapNestedForallToThreads::applyToOne( |
| 920 | transform::TransformRewriter &rewriter, Operation *target, |
| 921 | ApplyToEachResultList &results, TransformState &state) { |
| 922 | LaunchOp gpuLaunch = dyn_cast<LaunchOp>(Val: target); |
| 923 | auto transformOp = cast<TransformOpInterface>(Val: getOperation()); |
| 924 | |
| 925 | // Basic high-level verifications. |
| 926 | if (!gpuLaunch) |
| 927 | return emitSilenceableError() << "Given target is not a gpu.launch" ; |
| 928 | |
| 929 | // Mapping to block ids. |
| 930 | SmallVector<int64_t> blockDims{getBlockDims()}; |
| 931 | DiagnosedSilenceableFailure diag = |
| 932 | checkGpuLimits(transformOp, gridDimX: std::nullopt, gridDimY: std::nullopt, gridDimZ: std::nullopt, |
| 933 | blockDimX: blockDims[0], blockDimY: blockDims[1], blockDimZ: blockDims[2]); |
| 934 | if (diag.isSilenceableFailure()) { |
| 935 | diag.attachNote(loc: getLoc()) << getBlockDimsAttrName() << " is too large" ; |
| 936 | return diag; |
| 937 | } |
| 938 | |
| 939 | // Set the GPU launch configuration for the block dims early, this is not |
| 940 | // subject to IR inspection. |
| 941 | diag = alterGpuLaunch(rewriter, gpuLaunch, transformOp, gridDimX: std::nullopt, |
| 942 | gridDimY: std::nullopt, gridDimZ: std::nullopt, blockDimX: blockDims[0], blockDimY: blockDims[1], |
| 943 | blockDimZ: blockDims[2]); |
| 944 | |
| 945 | rewriter.setInsertionPointToStart(&gpuLaunch.getBody().front()); |
| 946 | diag = |
| 947 | mapNestedForallToThreadsImpl(rewriter, transformOp, target: gpuLaunch, blockDims, |
| 948 | warpSize: getWarpSize(), syncAfterDistribute: getSyncAfterDistribute()); |
| 949 | |
| 950 | results.push_back(op: gpuLaunch.getOperation()); |
| 951 | return diag; |
| 952 | } |
| 953 | |
| 954 | //===----------------------------------------------------------------------===// |
| 955 | // Transform op registration |
| 956 | //===----------------------------------------------------------------------===// |
| 957 | |
| 958 | namespace { |
| 959 | /// Registers new ops and declares PDL as dependent dialect since the |
| 960 | /// additional ops are using PDL types for operands and results. |
| 961 | class GPUTransformDialectExtension |
| 962 | : public transform::TransformDialectExtension< |
| 963 | GPUTransformDialectExtension> { |
| 964 | public: |
| 965 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(GPUTransformDialectExtension) |
| 966 | |
| 967 | GPUTransformDialectExtension() { |
| 968 | declareGeneratedDialect<GPUDialect>(); |
| 969 | declareGeneratedDialect<amdgpu::AMDGPUDialect>(); |
| 970 | declareGeneratedDialect<arith::ArithDialect>(); |
| 971 | declareGeneratedDialect<scf::SCFDialect>(); |
| 972 | registerTransformOps< |
| 973 | #define GET_OP_LIST |
| 974 | #include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.cpp.inc" |
| 975 | >(); |
| 976 | } |
| 977 | }; |
| 978 | } // namespace |
| 979 | |
| 980 | #define GET_OP_CLASSES |
| 981 | #include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.cpp.inc" |
| 982 | |
| 983 | void mlir::gpu::registerTransformDialectExtension(DialectRegistry ®istry) { |
| 984 | registry.addExtensions<GPUTransformDialectExtension>(); |
| 985 | } |
| 986 | |