1//===- GPUTransformOps.cpp - Implementation of GPU transform ops ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.h"
10
11#include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
12#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
13#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
14#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
15#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
16#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/GPU/IR/GPUDialect.h"
19#include "mlir/Dialect/GPU/TransformOps/Utils.h"
20#include "mlir/Dialect/GPU/Transforms/Passes.h"
21#include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h"
22#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
23#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
24#include "mlir/Dialect/MemRef/IR/MemRef.h"
25#include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h"
26#include "mlir/Dialect/SCF/IR/SCF.h"
27#include "mlir/Dialect/Transform/IR/TransformDialect.h"
28#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
29#include "mlir/Dialect/Utils/IndexingUtils.h"
30#include "mlir/Dialect/Vector/IR/VectorOps.h"
31#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
32#include "mlir/IR/AffineExpr.h"
33#include "mlir/IR/Builders.h"
34#include "mlir/IR/BuiltinAttributes.h"
35#include "mlir/IR/IRMapping.h"
36#include "mlir/IR/MLIRContext.h"
37#include "mlir/IR/OpDefinition.h"
38#include "mlir/IR/Visitors.h"
39#include "mlir/Support/LLVM.h"
40#include "mlir/Transforms/DialectConversion.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/ADT/TypeSwitch.h"
44#include "llvm/Support/DebugLog.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/InterleavedRange.h"
47#include "llvm/Support/LogicalResult.h"
48#include <optional>
49#include <type_traits>
50
51using namespace mlir;
52using namespace mlir::gpu;
53using namespace mlir::transform;
54using namespace mlir::transform::gpu;
55
56#define DEBUG_TYPE "gpu-transforms"
57
58//===----------------------------------------------------------------------===//
59// Apply...ConversionPatternsOp
60//===----------------------------------------------------------------------===//
61
62void transform::ApplyGPUToNVVMConversionPatternsOp::populatePatterns(
63 TypeConverter &typeConverter, RewritePatternSet &patterns) {
64 auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
65 configureGpuToNVVMTypeConverter(converter&: llvmTypeConverter);
66 // Set higher benefit, so patterns will run before generic LLVM lowering.
67 populateGpuToNVVMConversionPatterns(converter: llvmTypeConverter, patterns,
68 benefit: getBenefit());
69}
70
71LogicalResult
72transform::ApplyGPUToNVVMConversionPatternsOp::verifyTypeConverter(
73 transform::TypeConverterBuilderOpInterface builder) {
74 if (builder.getTypeConverterType() != "LLVMTypeConverter")
75 return emitOpError(message: "expected LLVMTypeConverter");
76 return success();
77}
78
79void transform::ApplyGPUWwmaToNVVMConversionPatternsOp::populatePatterns(
80 TypeConverter &typeConverter, RewritePatternSet &patterns) {
81 auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
82 populateGpuWMMAToNVVMConversionPatterns(converter: llvmTypeConverter, patterns);
83}
84
85LogicalResult
86transform::ApplyGPUWwmaToNVVMConversionPatternsOp::verifyTypeConverter(
87 transform::TypeConverterBuilderOpInterface builder) {
88 if (builder.getTypeConverterType() != "LLVMTypeConverter")
89 return emitOpError(message: "expected LLVMTypeConverter");
90 return success();
91}
92
93void transform::ApplyGPUSubgroupReduceToNVVMConversionPatternsOp::
94 populatePatterns(TypeConverter &typeConverter,
95 RewritePatternSet &patterns) {
96 auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
97 populateGpuSubgroupReduceOpLoweringPattern(converter: llvmTypeConverter, patterns);
98}
99
100LogicalResult transform::ApplyGPUSubgroupReduceToNVVMConversionPatternsOp::
101 verifyTypeConverter(transform::TypeConverterBuilderOpInterface builder) {
102 if (builder.getTypeConverterType() != "LLVMTypeConverter")
103 return emitOpError(message: "expected LLVMTypeConverter");
104 return success();
105}
106
107void transform::ApplyGPUToROCDLConversionPatternsOp::populatePatterns(
108 TypeConverter &typeConverter, RewritePatternSet &patterns) {
109 auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
110 amdgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: llvmTypeConverter);
111 // The verifier has already rejected anything unparseable.
112 FailureOr<ROCDL::TargetInfo> targetInfo =
113 ROCDL::TargetInfo::get(arch: getArch(), waveSize: getWavesize().value_or(u: 0));
114 assert(llvm::succeeded(targetInfo) && "verifier accepted this target");
115 populateGpuToROCDLConversionPatterns(
116 converter: llvmTypeConverter, patterns, runtime: mlir::gpu::amd::Runtime::HIP, target: *targetInfo);
117}
118
119LogicalResult
120transform::ApplyGPUToROCDLConversionPatternsOp::verifyTypeConverter(
121 transform::TypeConverterBuilderOpInterface builder) {
122 if (failed(Result: ROCDL::TargetInfo::get(arch: getArch(), waveSize: getWavesize().value_or(u: 0),
123 emitError: [&] { return emitOpError(); })))
124 return failure();
125 if (builder.getTypeConverterType() != "LLVMTypeConverter")
126 return emitOpError(message: "expected LLVMTypeConverter");
127 return success();
128}
129
130//===----------------------------------------------------------------------===//
131// Apply...PatternsOp
132//===----------------------------------------------------------------------===//s
133
134void ApplyGPURewritePatternsOp::populatePatterns(RewritePatternSet &patterns) {
135 populateGpuRewritePatterns(patterns);
136}
137
138void transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::populatePatterns(
139 RewritePatternSet &patterns) {
140 std::optional<StringRef> archName = getArch();
141 std::optional<ROCDL::TargetInfo> targetInfo;
142 if (archName) {
143 // The verifier has already rejected anything unparseable.
144 FailureOr<ROCDL::TargetInfo> parsed = ROCDL::TargetInfo::get(arch: *archName);
145 assert(llvm::succeeded(parsed) && "verifier accepted this target");
146 targetInfo = *parsed;
147 }
148
149 populateGpuPromoteShuffleToAMDGPUPatterns(patterns, target: targetInfo);
150}
151
152LogicalResult transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::verify() {
153 std::optional<StringRef> archName = getArch();
154 if (!archName)
155 return success();
156
157 if (failed(Result: ROCDL::TargetInfo::get(arch: *archName, /*waveSize=*/0,
158 emitError: [&] { return emitOpError(); })))
159 return failure();
160 return success();
161}
162
163//===----------------------------------------------------------------------===//
164// ApplyUnrollVectorsSubgroupMmaOp
165//===----------------------------------------------------------------------===//
166
167/// Pick an unrolling order that will allow tensorcore operation to reuse LHS
168/// register.
169static std::optional<SmallVector<int64_t>>
170gpuMmaUnrollOrder(vector::ContractionOp contract) {
171 SmallVector<int64_t> order;
172 // First make reduction the outer dimensions.
173 for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) {
174 if (vector::isReductionIterator(attr: iter)) {
175 order.push_back(Elt: index);
176 }
177 }
178
179 llvm::SmallDenseSet<int64_t> dims;
180 for (AffineExpr expr : contract.getIndexingMapsArray()[0].getResults()) {
181 dims.insert(V: cast<AffineDimExpr>(Val&: expr).getPosition());
182 }
183 // Then parallel dimensions that are part of Lhs as we want to re-use Lhs.
184 for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) {
185 if (vector::isParallelIterator(attr: iter) && dims.count(V: index)) {
186 order.push_back(Elt: index);
187 }
188 }
189 // Then the remaining parallel loops.
190 for (auto [index, iter] : llvm::enumerate(First: contract.getIteratorTypes())) {
191 if (vector::isParallelIterator(attr: iter) && !dims.count(V: index)) {
192 order.push_back(Elt: index);
193 }
194 }
195 return order;
196}
197
198/// Returns the target vector size for the target operation based on the native
199/// vector size specified with `m`, `n`, and `k`.
200static std::optional<SmallVector<int64_t>>
201getSubgroupMmaNativeVectorSize(Operation *op, int64_t m, int64_t n, int64_t k) {
202 if (auto contract = dyn_cast<vector::ContractionOp>(Val: op)) {
203 int64_t contractRank = contract.getIteratorTypes().size();
204 if (contractRank < 3)
205 return std::nullopt;
206 SmallVector<int64_t> nativeSize(contractRank - 3, 1);
207 nativeSize.append(IL: {m, n, k});
208 return nativeSize;
209 }
210 if (auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: op)) {
211 int64_t writeRank = writeOp.getVectorType().getRank();
212 if (writeRank < 2)
213 return std::nullopt;
214 SmallVector<int64_t> nativeSize(writeRank - 2, 1);
215 nativeSize.append(IL: {m, n});
216 return nativeSize;
217 }
218 if (auto readOp = dyn_cast<vector::TransferReadOp>(Val: op)) {
219 // Transfer read ops may need different shapes based on how they are being
220 // used. For simplicity just match the shape used by the extract strided op.
221 VectorType sliceType;
222 for (Operation *users : op->getUsers()) {
223 auto extract = dyn_cast<vector::ExtractStridedSliceOp>(Val: users);
224 if (!extract)
225 return std::nullopt;
226 auto vecType = cast<VectorType>(Val: extract.getResult().getType());
227 if (sliceType && sliceType != vecType)
228 return std::nullopt;
229 sliceType = vecType;
230 }
231 return llvm::to_vector(Range: sliceType.getShape());
232 }
233 if ((OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1)) {
234 if (auto vecType = dyn_cast<VectorType>(Val: op->getResultTypes()[0])) {
235 // TODO: The condition for unrolling elementwise should be restricted
236 // only to operations that need unrolling (connected to the contract).
237 if (vecType.getRank() < 2)
238 return std::nullopt;
239
240 // First check whether there is a slice to infer the shape from. This is
241 // required for cases where the accumulator type differs from the input
242 // types, in which case we will see an `arith.ext_` between the contract
243 // and transfer_read which needs to be unrolled.
244 VectorType sliceType;
245 for (Operation *users : op->getUsers()) {
246 auto extract = dyn_cast<vector::ExtractStridedSliceOp>(Val: users);
247 if (!extract)
248 return std::nullopt;
249 auto vecType = cast<VectorType>(Val: extract.getResult().getType());
250 if (sliceType && sliceType != vecType)
251 return std::nullopt;
252 sliceType = vecType;
253 }
254 if (sliceType)
255 return llvm::to_vector(Range: sliceType.getShape());
256
257 // Else unroll for trailing elementwise.
258 SmallVector<int64_t> nativeSize(vecType.getRank() - 2, 1);
259 // Map elementwise ops to the output shape.
260 nativeSize.append(IL: {m, n});
261 return nativeSize;
262 }
263 }
264 return std::nullopt;
265}
266
267void transform::ApplyUnrollVectorsSubgroupMmaOp::populatePatterns(
268 RewritePatternSet &patterns) {
269 auto unrollOrder = [](Operation *op) -> std::optional<SmallVector<int64_t>> {
270 auto contract = dyn_cast<vector::ContractionOp>(Val: op);
271 if (!contract)
272 return std::nullopt;
273 return gpuMmaUnrollOrder(contract);
274 };
275
276 int64_t m = getM();
277 int64_t n = getN();
278 int64_t k = getK();
279 auto nativeShapeFn =
280 [m, n, k](Operation *op) -> std::optional<SmallVector<int64_t>> {
281 return getSubgroupMmaNativeVectorSize(op, m, n, k);
282 };
283 vector::populateVectorUnrollPatterns(
284 patterns, options: vector::UnrollVectorOptions()
285 .setNativeShapeFn(nativeShapeFn)
286 .setUnrollTraversalOrderFn(unrollOrder));
287}
288
289//===----------------------------------------------------------------------===//
290// EliminateBarriersOp
291//===----------------------------------------------------------------------===//
292
293void EliminateBarriersOp::populatePatterns(RewritePatternSet &patterns) {
294 populateGpuEliminateBarriersPatterns(patterns);
295}
296
297//===----------------------------------------------------------------------===//
298// Block and thread mapping utilities.
299//===----------------------------------------------------------------------===//
300
301namespace {
302/// Local types used for mapping verification.
303struct MappingKind {};
304struct BlockMappingKind : MappingKind {};
305struct ThreadMappingKind : MappingKind {};
306} // namespace
307
308static DiagnosedSilenceableFailure
309definiteFailureHelper(std::optional<TransformOpInterface> transformOp,
310 Operation *target, const Twine &message) {
311 if (transformOp.has_value())
312 return transformOp->emitDefiniteFailure() << message;
313 return emitDefiniteFailure(op: target, message);
314}
315
316/// Check if given mapping attributes are one of the desired attributes
317template <typename MappingKindType>
318static DiagnosedSilenceableFailure
319checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
320 scf::ForallOp forallOp) {
321 if (!forallOp.getMapping().has_value()) {
322 return definiteFailureHelper(transformOp, target: forallOp,
323 message: "scf.forall op requires a mapping attribute");
324 }
325
326 bool hasBlockMapping = llvm::any_of(Range: forallOp.getMapping().value(),
327 P: llvm::IsaPred<GPUBlockMappingAttr>);
328 bool hasWarpgroupMapping = llvm::any_of(
329 Range: forallOp.getMapping().value(), P: llvm::IsaPred<GPUWarpgroupMappingAttr>);
330 bool hasWarpMapping = llvm::any_of(Range: forallOp.getMapping().value(),
331 P: llvm::IsaPred<GPUWarpMappingAttr>);
332 bool hasThreadMapping = llvm::any_of(Range: forallOp.getMapping().value(),
333 P: llvm::IsaPred<GPUThreadMappingAttr>);
334 bool hasLaneMapping = llvm::any_of(Range: forallOp.getMapping().value(),
335 P: llvm::IsaPred<GPULaneMappingAttr>);
336 int64_t countMappingTypes = 0;
337 countMappingTypes += hasBlockMapping ? 1 : 0;
338 countMappingTypes += hasWarpgroupMapping ? 1 : 0;
339 countMappingTypes += hasWarpMapping ? 1 : 0;
340 countMappingTypes += hasThreadMapping ? 1 : 0;
341 countMappingTypes += hasLaneMapping ? 1 : 0;
342 if (countMappingTypes > 1) {
343 return definiteFailureHelper(
344 transformOp, target: forallOp,
345 message: "cannot mix different mapping types, use nesting");
346 }
347 if (std::is_same<MappingKindType, BlockMappingKind>::value &&
348 !hasBlockMapping) {
349 return definiteFailureHelper(
350 transformOp, target: forallOp,
351 message: "scf.forall op requires a mapping attribute of kind 'block'");
352 }
353 if (std::is_same<MappingKindType, ThreadMappingKind>::value &&
354 !hasLaneMapping && !hasThreadMapping && !hasWarpMapping &&
355 !hasWarpgroupMapping) {
356 return definiteFailureHelper(transformOp, target: forallOp,
357 message: "scf.forall op requires a mapping attribute "
358 "of kind 'thread' or 'warp'");
359 }
360
361 DenseSet<Attribute> seen;
362 for (Attribute map : forallOp.getMapping()->getValue()) {
363 if (seen.contains(V: map)) {
364 return definiteFailureHelper(
365 transformOp, target: forallOp,
366 message: "duplicate attribute, cannot map different loops "
367 "to the same mapping id");
368 }
369 seen.insert(V: map);
370 }
371
372 auto isLinear = [](DeviceMappingAttrInterface attr) {
373 return attr.isLinearMapping();
374 };
375 if (llvm::any_of(forallOp.getDeviceMappingAttrs(), isLinear) &&
376 !llvm::all_of(forallOp.getDeviceMappingAttrs(), isLinear)) {
377 return definiteFailureHelper(
378 transformOp, target: forallOp,
379 message: "cannot mix linear and non-linear mapping modes");
380 }
381
382 FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
383 forallOp.getDeviceMaskingAttr();
384 if (succeeded(Result: maybeMaskingAttr) && *maybeMaskingAttr &&
385 !forallOp.usesLinearMapping()) {
386 return definiteFailureHelper(
387 transformOp, target: forallOp,
388 message: "device masking is only available in linear mapping mode");
389 }
390
391 return DiagnosedSilenceableFailure::success();
392}
393
394template <typename MappingKindType>
395static DiagnosedSilenceableFailure
396verifyGpuMapping(std::optional<TransformOpInterface> transformOp,
397 scf::ForallOp forallOp) {
398 // Check the types of the mapping attributes match.
399 DiagnosedSilenceableFailure typeRes =
400 checkMappingAttributeTypes<MappingKindType>(transformOp, forallOp);
401 if (!typeRes.succeeded())
402 return typeRes;
403
404 // Perform other non-types verifications.
405 if (!forallOp.isNormalized())
406 return definiteFailureHelper(transformOp, target: forallOp,
407 message: "unsupported non-normalized loops");
408 if (forallOp.getNumResults() > 0)
409 return definiteFailureHelper(transformOp, target: forallOp,
410 message: "only bufferized scf.forall can be mapped");
411 bool useLinearMapping = forallOp.usesLinearMapping();
412 // TODO: This would be more natural with support for Optional<EnumParameter>
413 // in GPUDeviceMappingAttr.
414 int64_t maxNumMappingsSupported =
415 useLinearMapping ? (getMaxEnumValForMappingId() -
416 static_cast<uint64_t>(MappingId::DimZ))
417 : 3;
418 if (forallOp.getRank() > maxNumMappingsSupported) {
419 return definiteFailureHelper(transformOp, target: forallOp,
420 message: "scf.forall with rank > ")
421 << maxNumMappingsSupported
422 << " does not lower for the specified mapping attribute type";
423 }
424 auto numParallelIterations =
425 getConstantIntValues(ofrs: forallOp.getMixedUpperBound());
426 if (!forallOp.isNormalized() || !numParallelIterations.has_value()) {
427 return definiteFailureHelper(
428 transformOp, target: forallOp,
429 message: "requires statically sized, normalized forall op");
430 }
431 return DiagnosedSilenceableFailure::success();
432}
433
434/// Struct to return the result of the rewrite of a forall operation.
435struct ForallRewriteResult {
436 SmallVector<int64_t> mappingSizes;
437 SmallVector<Value> mappingIds;
438};
439
440/// Helper to replace ids of dimensions known to be 1 by 0 to simplify the IR.
441template <typename OpTy, typename OperationOrBlock>
442static void
443replaceUnitMappingIdsHelper(RewriterBase &rewriter, Location loc,
444 OperationOrBlock *parent, Value replacement,
445 ArrayRef<int64_t> availableMappingSizes) {
446 parent->walk([&](OpTy idOp) {
447 if (availableMappingSizes[static_cast<int64_t>(idOp.getDimension())] == 1)
448 rewriter.replaceAllUsesWith(idOp.getResult(), replacement);
449 });
450}
451
452static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
453 RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp,
454 scf::ForallOp forallOp, ArrayRef<int64_t> availableMappingSizes,
455 ForallRewriteResult &result, const GpuIdBuilder &gpuIdBuilder) {
456 LDBG() << "--start rewriteOneForallCommonImpl";
457
458 // Step 1. Complete the mapping to a full mapping (with 1s) if necessary.
459 auto numParallelIterations =
460 getConstantIntValues(ofrs: forallOp.getMixedUpperBound());
461 assert(forallOp.isNormalized() && numParallelIterations.has_value() &&
462 "requires statically sized, normalized forall op");
463 SmallVector<int64_t> tmpMappingSizes = numParallelIterations.value();
464 SmallVector<DeviceMappingAttrInterface> forallMappingAttrsVec =
465 forallOp.getDeviceMappingAttrs();
466 SetVector<Attribute> forallMappingAttrs;
467 forallMappingAttrs.insert_range(R&: forallMappingAttrsVec);
468 auto comparator = [](Attribute a, Attribute b) -> bool {
469 return cast<DeviceMappingAttrInterface>(Val&: a).getMappingId() <
470 cast<DeviceMappingAttrInterface>(Val&: b).getMappingId();
471 };
472
473 // Step 1.b. In the linear case, compute the max mapping to avoid needlessly
474 // mapping all dimensions. In the 3-D mapping case we need to map all
475 // dimensions.
476 DeviceMappingAttrInterface maxMapping = cast<DeviceMappingAttrInterface>(
477 Val: *llvm::max_element(Range&: forallMappingAttrs, C: comparator));
478 DeviceMappingAttrInterface maxLinearMapping;
479 if (maxMapping.isLinearMapping())
480 maxLinearMapping = maxMapping;
481 for (auto attr : gpuIdBuilder.mappingAttributes) {
482 // If attr overflows, just skip.
483 if (maxLinearMapping && comparator(maxLinearMapping, attr))
484 continue;
485 // Try to insert. If element was already present, just continue.
486 if (!forallMappingAttrs.insert(X: attr))
487 continue;
488 // Otherwise, we have a new insertion without a size -> use size 1.
489 tmpMappingSizes.push_back(Elt: 1);
490 }
491 LDBG() << "----tmpMappingSizes extracted from scf.forall op: "
492 << llvm::interleaved(R: tmpMappingSizes);
493
494 // Step 2. sort the values by the corresponding DeviceMappingAttrInterface.
495 SmallVector<int64_t> forallMappingSizes = getValuesSortedByKey(
496 keys: forallMappingAttrs.getArrayRef(), values: tmpMappingSizes, compare: comparator);
497 LDBG() << "----forallMappingSizes: " << llvm::interleaved(R: forallMappingSizes);
498 LDBG() << "----forallMappingAttrs: " << llvm::interleaved(R: forallMappingAttrs);
499
500 // Step 3. Generate the mappingIdOps using the provided generator.
501 Location loc = forallOp.getLoc();
502 OpBuilder::InsertionGuard guard(rewriter);
503 rewriter.setInsertionPoint(forallOp);
504 SmallVector<int64_t> originalBasis(availableMappingSizes);
505 bool originalBasisWasProvided = !originalBasis.empty();
506 if (!originalBasisWasProvided) {
507 LDBG() << "----originalBasis was not provided, deriving it and there will "
508 "be no "
509 "predication";
510 originalBasis = forallMappingSizes;
511 while (originalBasis.size() < 3)
512 originalBasis.push_back(Elt: 1);
513 } else {
514 LDBG() << "----originalBasis was provided, using it, there will be "
515 "predication";
516 }
517 LDBG() << "------originalBasis: " << llvm::interleaved(R: originalBasis);
518
519 IdBuilderResult builderResult =
520 gpuIdBuilder.idBuilder(rewriter, loc, forallMappingSizes, originalBasis);
521 if (!builderResult.errorMsg.empty())
522 return definiteFailureHelper(transformOp, target: forallOp, message: builderResult.errorMsg);
523
524 LDBG() << builderResult;
525
526 // Step 4. Map the induction variables to the mappingIdOps, this may involve
527 // a permutation.
528 SmallVector<Value> mappingIdOps = builderResult.mappingIdOps;
529 IRMapping bvm;
530 for (auto [iv, dim] : llvm::zip_equal(
531 t: forallOp.getInductionVars(),
532 u: forallMappingAttrs.getArrayRef().take_front(N: forallOp.getRank()))) {
533 auto mappingAttr = cast<DeviceMappingAttrInterface>(Val: dim);
534 Value peIdOp = mappingIdOps[mappingAttr.getRelativeIndex()];
535 LDBG() << "----map: " << iv << " to " << peIdOp;
536 bvm.map(from: iv, to: peIdOp);
537 }
538
539 // Step 5. If the originalBasis is already known, create conditionals to
540 // predicate the region. Otherwise, the current forall determines the
541 // originalBasis and no predication occurs.
542 Value predicate;
543 if (originalBasisWasProvided) {
544 for (Value tmpPredicate : builderResult.predicateOps) {
545 predicate = predicate ? arith::AndIOp::create(builder&: rewriter, location: loc, lhs: predicate,
546 rhs: tmpPredicate)
547 : tmpPredicate;
548 }
549 }
550
551 // Step 6. Move the body of forallOp.
552 // Erase the terminator first, it will not be used.
553 rewriter.eraseOp(op: forallOp.getTerminator());
554 Block *targetBlock;
555 Block::iterator insertionPoint;
556 if (predicate) {
557 // Step 6.a. If predicated, move at the beginning.
558 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate,
559 /*withElseRegion=*/false);
560 targetBlock = ifOp.thenBlock();
561 insertionPoint = ifOp.thenBlock()->begin();
562 } else {
563 // Step 6.b. Otherwise, move inline just at the rewriter insertion
564 // point.
565 targetBlock = forallOp->getBlock();
566 insertionPoint = rewriter.getInsertionPoint();
567 }
568 Block &sourceBlock = forallOp.getRegion().front();
569 targetBlock->getOperations().splice(where: insertionPoint,
570 L2&: sourceBlock.getOperations());
571
572 // Step 7. RAUW indices.
573 for (Value loopIndex : forallOp.getInductionVars()) {
574 Value threadIdx = bvm.lookup(from: loopIndex);
575 rewriter.replaceAllUsesWith(from: loopIndex, to: threadIdx);
576 }
577
578 // Step 8. Erase old op.
579 rewriter.eraseOp(op: forallOp);
580
581 LDBG() << "----result forallMappingSizes: "
582 << llvm::interleaved(R: forallMappingSizes);
583 LDBG() << "----result mappingIdOps: " << llvm::interleaved(R: mappingIdOps);
584
585 result = ForallRewriteResult{.mappingSizes: forallMappingSizes, .mappingIds: mappingIdOps};
586 return DiagnosedSilenceableFailure::success();
587}
588
589//===----------------------------------------------------------------------===//
590// MapForallToBlocks
591//===----------------------------------------------------------------------===//
592
593DiagnosedSilenceableFailure mlir::transform::gpu::mapForallToBlocksImpl(
594 RewriterBase &rewriter, TransformOpInterface transformOp,
595 scf::ForallOp forallOp, SmallVectorImpl<int64_t> &gridDims,
596 const GpuIdBuilder &gpuIdBuilder) {
597 LDBG() << "Start mapForallToBlocksImpl";
598
599 {
600 // GPU-specific verifications. There is no better place to anchor
601 // those right now: the ForallOp is target-independent and the transform
602 // op does not apply to individual ForallOp.
603 DiagnosedSilenceableFailure diag =
604 verifyGpuMapping<BlockMappingKind>(transformOp, forallOp);
605 if (!diag.succeeded())
606 return diag;
607 }
608
609 Location loc = forallOp.getLoc();
610 Block *parentBlock = forallOp->getBlock();
611 Value zero;
612 {
613 // Create an early zero index value for replacements and immediately reset
614 // the insertion point.
615 OpBuilder::InsertionGuard guard(rewriter);
616 rewriter.setInsertionPointToStart(parentBlock);
617 zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
618 }
619
620 ForallRewriteResult rewriteResult;
621 DiagnosedSilenceableFailure diag = rewriteOneForallCommonImpl(
622 rewriter, transformOp, forallOp,
623 /*availableMappingSizes=*/gridDims, result&: rewriteResult, gpuIdBuilder);
624
625 // Return if anything goes wrong, use silenceable failure as a match
626 // failure.
627 if (!diag.succeeded())
628 return diag;
629
630 // If gridDims was not provided already, set it from the return.
631 if (gridDims.empty()) {
632 gridDims = rewriteResult.mappingSizes;
633 while (gridDims.size() < 3)
634 gridDims.push_back(Elt: 1);
635 }
636 assert(gridDims.size() == 3 && "Need 3-D gridDims");
637
638 // Replace ids of dimensions known to be 1 by 0 to simplify the IR.
639 // Here, the result of mapping determines the available mapping sizes.
640 replaceUnitMappingIdsHelper<BlockDimOp>(rewriter, loc, parent: parentBlock, replacement: zero,
641 availableMappingSizes: rewriteResult.mappingSizes);
642
643 return DiagnosedSilenceableFailure::success();
644}
645
646DiagnosedSilenceableFailure
647mlir::transform::gpu::findTopLevelForallOp(Operation *target,
648 scf::ForallOp &topLevelForallOp,
649 TransformOpInterface transformOp) {
650 auto walkResult = target->walk(callback: [&](scf::ForallOp forallOp) {
651 if (forallOp->getParentOfType<scf::ForallOp>())
652 return WalkResult::advance();
653 if (topLevelForallOp)
654 // TODO: Handle multiple forall if they are independent.
655 return WalkResult::interrupt();
656 topLevelForallOp = forallOp;
657 return WalkResult::advance();
658 });
659
660 if (walkResult.wasInterrupted() || !topLevelForallOp)
661 return transformOp.emitSilenceableError()
662 << "could not find a unique topLevel scf.forall";
663 return DiagnosedSilenceableFailure::success();
664}
665
666DiagnosedSilenceableFailure transform::MapForallToBlocks::applyToOne(
667 transform::TransformRewriter &rewriter, Operation *target,
668 ApplyToEachResultList &results, transform::TransformState &state) {
669 LaunchOp gpuLaunch = dyn_cast<LaunchOp>(Val: target);
670 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
671
672 if (!getGenerateGpuLaunch() && !gpuLaunch) {
673 DiagnosedSilenceableFailure diag =
674 emitSilenceableError()
675 << "Given target is not gpu.launch, set `generate_gpu_launch` "
676 "attribute";
677 diag.attachNote(loc: target->getLoc()) << "when applied to this payload op";
678 return diag;
679 }
680
681 scf::ForallOp topLevelForallOp;
682 DiagnosedSilenceableFailure diag = mlir::transform::gpu::findTopLevelForallOp(
683 target, topLevelForallOp, transformOp);
684 if (!diag.succeeded()) {
685 diag.attachNote(loc: target->getLoc()) << "when applied to this payload op";
686 return diag;
687 }
688 assert(topLevelForallOp && "expect an scf.forall");
689
690 SmallVector<int64_t> gridDims{getGridDims()};
691 if (!getGenerateGpuLaunch() && gridDims.size() != 3)
692 return transformOp.emitDefiniteFailure(message: "transform require size-3 mapping");
693
694 OpBuilder::InsertionGuard guard(rewriter);
695 rewriter.setInsertionPoint(topLevelForallOp);
696
697 // Generate gpu launch here and move the forall inside
698 if (getGenerateGpuLaunch()) {
699 DiagnosedSilenceableFailure diag =
700 createGpuLaunch(rewriter, loc: target->getLoc(), transformOp, launchOp&: gpuLaunch);
701 if (!diag.succeeded())
702 return diag;
703
704 rewriter.setInsertionPointToStart(&gpuLaunch.getBody().front());
705 Operation *newForallOp = rewriter.clone(op&: *topLevelForallOp);
706 rewriter.eraseOp(op: topLevelForallOp);
707 topLevelForallOp = cast<scf::ForallOp>(Val: newForallOp);
708 }
709
710 // The BlockIdBuilder adapts to whatever is thrown at it.
711 bool useLinearMapping = false;
712 if (topLevelForallOp.getMapping())
713 useLinearMapping = topLevelForallOp.usesLinearMapping();
714
715 FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
716 topLevelForallOp.getDeviceMaskingAttr();
717 assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr");
718 assert((!*maybeMaskingAttr || useLinearMapping) &&
719 "masking requires linear mapping");
720
721 GpuBlockIdBuilder gpuBlockIdBuilder(getContext(), useLinearMapping,
722 *maybeMaskingAttr);
723
724 diag = mlir::transform::gpu::mapForallToBlocksImpl(
725 rewriter, transformOp, forallOp: topLevelForallOp, gridDims, gpuIdBuilder: gpuBlockIdBuilder);
726 if (!diag.succeeded())
727 return diag;
728
729 // Set the GPU launch configuration for the grid dims late, this is
730 // subject to IR inspection.
731 diag = alterGpuLaunch(rewriter, gpuLaunch,
732 transformOp: cast<TransformOpInterface>(Val: getOperation()), gridDimX: gridDims[0],
733 gridDimY: gridDims[1], gridDimZ: gridDims[2]);
734
735 results.push_back(op: gpuLaunch);
736 return diag;
737}
738
739LogicalResult transform::MapForallToBlocks::verify() {
740 if (!getGridDims().empty() && getGridDims().size() != 3) {
741 return emitOpError() << "transform requires empty or size-3 grid_dims";
742 }
743 return success();
744}
745
746//===----------------------------------------------------------------------===//
747// MapNestedForallToThreads
748//===----------------------------------------------------------------------===//
749
750static DiagnosedSilenceableFailure checkMappingSpec(
751 std::optional<TransformOpInterface> transformOp, scf::ForallOp forallOp,
752 ArrayRef<int64_t> numParallelIterations, ArrayRef<int64_t> blockOrGridSizes,
753 int factor, bool useLinearMapping = false) {
754 if (llvm::any_of(Range&: blockOrGridSizes, P: [](int64_t i) { return i <= 0; })) {
755 return definiteFailureHelper(transformOp, target: forallOp,
756 message: "block/grid sizes must be strictly positive");
757 }
758 if (!useLinearMapping && blockOrGridSizes.front() % factor != 0) {
759 auto diag = definiteFailureHelper(
760 transformOp, target: forallOp,
761 message: Twine("3-D mapping: size of threadIdx.x must be a multiple of ") +
762 Twine(factor));
763 return diag;
764 }
765 bool hasZeroParallelIteration =
766 llvm::any_of(Range&: numParallelIterations, P: [](int64_t i) { return i == 0; });
767 // `computeProduct` requires strictly positive inputs, so handle the
768 // zero-iteration case explicitly to avoid asserting on valid degenerate
769 // loop bounds.
770 int64_t requiredResourceCount =
771 hasZeroParallelIteration ? 0
772 : computeProduct(basis: numParallelIterations) * factor;
773 int64_t availableResourceCount = computeProduct(basis: blockOrGridSizes);
774 if (requiredResourceCount > availableResourceCount) {
775 auto diag = definiteFailureHelper(
776 transformOp, target: forallOp,
777 message: Twine("the number of required parallel resources (blocks or "
778 "threads) ") +
779 Twine(requiredResourceCount) +
780 " overflows the number of available resources " +
781 Twine(availableResourceCount));
782 return diag;
783 }
784 return DiagnosedSilenceableFailure::success();
785}
786
787static DiagnosedSilenceableFailure
788getThreadIdBuilder(std::optional<TransformOpInterface> transformOp,
789 scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes,
790 int64_t warpSize, GpuIdBuilder &gpuIdBuilder) {
791 DeviceMappingAttrInterface mappingAttr =
792 forallOp.getDeviceMappingAttrs().front();
793 bool useLinearMapping = mappingAttr.isLinearMapping();
794
795 // Sanity checks that may result in runtime verification errors.
796 auto numParallelIterations =
797 getConstantIntValues(ofrs: (forallOp.getMixedUpperBound()));
798 if (!forallOp.isNormalized() || !numParallelIterations.has_value()) {
799 return definiteFailureHelper(
800 transformOp, target: forallOp,
801 message: "requires statically sized, normalized forall op");
802 }
803 int64_t factor = 1;
804 if (isa<GPUWarpgroupMappingAttr>(Val: mappingAttr)) {
805 factor = GpuWarpgroupIdBuilder::kNumWarpsPerGroup * warpSize;
806 } else if (isa<GPUWarpMappingAttr>(Val: mappingAttr)) {
807 factor = warpSize;
808 }
809 DiagnosedSilenceableFailure diag =
810 checkMappingSpec(transformOp, forallOp, numParallelIterations: numParallelIterations.value(),
811 blockOrGridSizes: blockSizes, factor, useLinearMapping);
812 if (!diag.succeeded())
813 return diag;
814
815 FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
816 forallOp.getDeviceMaskingAttr();
817 assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr");
818 assert((!*maybeMaskingAttr || useLinearMapping) &&
819 "masking requires linear mapping");
820
821 // Start mapping.
822 MLIRContext *ctx = forallOp.getContext();
823 gpuIdBuilder =
824 TypeSwitch<DeviceMappingAttrInterface, GpuIdBuilder>(mappingAttr)
825 .Case(caseFn: [&](GPUWarpgroupMappingAttr) {
826 return GpuWarpgroupIdBuilder(ctx, warpSize, useLinearMapping,
827 *maybeMaskingAttr);
828 })
829 .Case(caseFn: [&](GPUWarpMappingAttr) {
830 return GpuWarpIdBuilder(ctx, warpSize, useLinearMapping,
831 *maybeMaskingAttr);
832 })
833 .Case(caseFn: [&](GPUThreadMappingAttr) {
834 return GpuThreadIdBuilder(ctx, useLinearMapping, *maybeMaskingAttr);
835 })
836 .Case(caseFn: [&](GPULaneMappingAttr) {
837 return GpuLaneIdBuilder(ctx, warpSize, useLinearMapping,
838 *maybeMaskingAttr);
839 })
840 .DefaultUnreachable(message: "unknown mapping attribute");
841 return DiagnosedSilenceableFailure::success();
842}
843
844DiagnosedSilenceableFailure mlir::transform::gpu::mapOneForallToThreadsImpl(
845 RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp,
846 scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes, int64_t warpSize,
847 bool syncAfterDistribute) {
848
849 {
850 // GPU-specific verifications. There is no better place to anchor
851 // those right now: the ForallOp is target-independent and the transform
852 // op does not apply to individual ForallOp.
853 DiagnosedSilenceableFailure diag =
854 verifyGpuMapping<ThreadMappingKind>(transformOp, forallOp);
855 if (!diag.succeeded())
856 return diag;
857 }
858
859 GpuIdBuilder gpuIdBuilder;
860 {
861 // Try to construct the id builder, if it fails, return.
862 DiagnosedSilenceableFailure diag = getThreadIdBuilder(
863 transformOp, forallOp, blockSizes, warpSize, gpuIdBuilder);
864 if (!diag.succeeded())
865 return diag;
866 }
867
868 Location loc = forallOp.getLoc();
869 OpBuilder::InsertionGuard g(rewriter);
870 // Insert after to allow for syncthreads after `forall` is erased.
871 rewriter.setInsertionPointAfter(forallOp);
872 ForallRewriteResult rewriteResult;
873 DiagnosedSilenceableFailure diag = rewriteOneForallCommonImpl(
874 rewriter, transformOp, forallOp, availableMappingSizes: blockSizes, result&: rewriteResult, gpuIdBuilder);
875 if (!diag.succeeded())
876 return diag;
877 // Add a syncthreads if needed. TODO: warpsync
878 if (syncAfterDistribute)
879 BarrierOp::create(builder&: rewriter, location: loc);
880
881 return DiagnosedSilenceableFailure::success();
882}
883
884DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
885 RewriterBase &rewriter, std::optional<TransformOpInterface> transformOp,
886 Operation *target, ArrayRef<int64_t> blockDims, int64_t warpSize,
887 bool syncAfterDistribute) {
888 LDBG() << "Start mapNestedForallToThreadsImpl";
889 if (blockDims.size() != 3) {
890 return definiteFailureHelper(transformOp, target,
891 message: "requires size-3 thread mapping");
892 }
893
894 // Create an early zero index value for replacements.
895 Location loc = target->getLoc();
896 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
897 DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
898 WalkResult walkResult = target->walk(callback: [&](scf::ForallOp forallOp) {
899 diag = mlir::transform::gpu::mapOneForallToThreadsImpl(
900 rewriter, transformOp, forallOp, blockSizes: blockDims, warpSize,
901 syncAfterDistribute);
902 if (diag.isDefiniteFailure())
903 return WalkResult::interrupt();
904 if (diag.succeeded())
905 return WalkResult::skip();
906 return WalkResult::advance();
907 });
908 if (walkResult.wasInterrupted())
909 return diag;
910
911 // Replace ids of dimensions known to be 1 by 0 to simplify the IR.
912 // Here, the result of mapping determines the available mapping sizes.
913 replaceUnitMappingIdsHelper<ThreadIdOp>(rewriter, loc, parent: target, replacement: zero,
914 availableMappingSizes: blockDims);
915
916 return DiagnosedSilenceableFailure::success();
917}
918
919DiagnosedSilenceableFailure transform::MapNestedForallToThreads::applyToOne(
920 transform::TransformRewriter &rewriter, Operation *target,
921 ApplyToEachResultList &results, TransformState &state) {
922 LaunchOp gpuLaunch = dyn_cast<LaunchOp>(Val: target);
923 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
924
925 // Basic high-level verifications.
926 if (!gpuLaunch)
927 return emitSilenceableError() << "Given target is not a gpu.launch";
928
929 // Mapping to block ids.
930 SmallVector<int64_t> blockDims{getBlockDims()};
931 DiagnosedSilenceableFailure diag =
932 checkGpuLimits(transformOp, gridDimX: std::nullopt, gridDimY: std::nullopt, gridDimZ: std::nullopt,
933 blockDimX: blockDims[0], blockDimY: blockDims[1], blockDimZ: blockDims[2]);
934 if (diag.isSilenceableFailure()) {
935 diag.attachNote(loc: getLoc()) << getBlockDimsAttrName() << " is too large";
936 return diag;
937 }
938
939 // Set the GPU launch configuration for the block dims early, this is not
940 // subject to IR inspection.
941 diag = alterGpuLaunch(rewriter, gpuLaunch, transformOp, gridDimX: std::nullopt,
942 gridDimY: std::nullopt, gridDimZ: std::nullopt, blockDimX: blockDims[0], blockDimY: blockDims[1],
943 blockDimZ: blockDims[2]);
944
945 rewriter.setInsertionPointToStart(&gpuLaunch.getBody().front());
946 diag =
947 mapNestedForallToThreadsImpl(rewriter, transformOp, target: gpuLaunch, blockDims,
948 warpSize: getWarpSize(), syncAfterDistribute: getSyncAfterDistribute());
949
950 results.push_back(op: gpuLaunch.getOperation());
951 return diag;
952}
953
954//===----------------------------------------------------------------------===//
955// Transform op registration
956//===----------------------------------------------------------------------===//
957
958namespace {
959/// Registers new ops and declares PDL as dependent dialect since the
960/// additional ops are using PDL types for operands and results.
961class GPUTransformDialectExtension
962 : public transform::TransformDialectExtension<
963 GPUTransformDialectExtension> {
964public:
965 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(GPUTransformDialectExtension)
966
967 GPUTransformDialectExtension() {
968 declareGeneratedDialect<GPUDialect>();
969 declareGeneratedDialect<amdgpu::AMDGPUDialect>();
970 declareGeneratedDialect<arith::ArithDialect>();
971 declareGeneratedDialect<scf::SCFDialect>();
972 registerTransformOps<
973#define GET_OP_LIST
974#include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.cpp.inc"
975 >();
976 }
977};
978} // namespace
979
980#define GET_OP_CLASSES
981#include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.cpp.inc"
982
983void mlir::gpu::registerTransformDialectExtension(DialectRegistry &registry) {
984 registry.addExtensions<GPUTransformDialectExtension>();
985}
986