1//===- PromoteShuffleToAMDGPU.cpp - Promote shuffle to AMDGPU -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains patterns to try to promote `gpu.shuffle`s to specialized
10// AMDGPU intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h"
15#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
16
17#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
18#include "mlir/Dialect/Arith/IR/Arith.h"
19#include "mlir/Dialect/GPU/IR/GPUDialect.h"
20#include "mlir/IR/PatternMatch.h"
21#include <optional>
22
23using namespace mlir;
24
25namespace {
26/// Try to promote `gpu.shuffle` to `amdgpu.swizzle_bitmode`, width must be 64
27/// and offset must be a constant integer in the range [0, 31].
28struct PromoteShuffleToSwizzlePattern
29 : public OpRewritePattern<gpu::ShuffleOp> {
30 using Base::Base;
31
32 LogicalResult matchAndRewrite(gpu::ShuffleOp op,
33 PatternRewriter &rewriter) const override {
34 if (op.getMode() != gpu::ShuffleMode::XOR)
35 return rewriter.notifyMatchFailure(arg&: op,
36 msg: "only xor shuffle mode is supported");
37
38 if (!isConstantIntValue(ofr: op.getWidth(), value: 64))
39 return rewriter.notifyMatchFailure(arg&: op,
40 msg: "only 64 width shuffle is supported");
41
42 std::optional<int64_t> offset = getConstantIntValue(ofr: op.getOffset());
43 if (!offset)
44 return rewriter.notifyMatchFailure(arg&: op,
45 msg: "offset must be a constant integer");
46
47 int64_t offsetValue = *offset;
48 if (offsetValue < 0 || offsetValue >= 32)
49 return rewriter.notifyMatchFailure(arg&: op,
50 msg: "offset must be in the range [0, 31]");
51
52 Location loc = op.getLoc();
53 Value res = amdgpu::SwizzleBitModeOp::create(
54 builder&: rewriter, location: loc, result: op.getResult(i: 0).getType(), src: op.getValue(),
55 /*and_mask=*/31,
56 /*orMask=*/or_mask: 0, /*xorMask=*/xor_mask: offsetValue);
57 Value valid = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 1, /*width*/ 1);
58 rewriter.replaceOp(op, newValues: {res, valid});
59 return success();
60 }
61};
62
63/// Try to promote `gpu.shuffle` to `amdgpu.permlane_swap`, width must be 64
64/// and offset must be a constant integer in the set {16, 32}.
65struct PromoteShuffleToPermlanePattern
66 : public OpRewritePattern<gpu::ShuffleOp> {
67 using Base::Base;
68
69 LogicalResult matchAndRewrite(gpu::ShuffleOp op,
70 PatternRewriter &rewriter) const override {
71 if (op.getMode() != gpu::ShuffleMode::XOR)
72 return rewriter.notifyMatchFailure(arg&: op,
73 msg: "only xor shuffle mode is supported");
74
75 if (!isConstantIntValue(ofr: op.getWidth(), value: 64))
76 return rewriter.notifyMatchFailure(arg&: op,
77 msg: "only 64 width shuffle is supported");
78
79 std::optional<int64_t> offset = getConstantIntValue(ofr: op.getOffset());
80 if (!offset)
81 return rewriter.notifyMatchFailure(arg&: op,
82 msg: "offset must be a constant integer");
83
84 int64_t offsetValue = *offset;
85 if (offsetValue != 16 && offsetValue != 32)
86 return rewriter.notifyMatchFailure(arg&: op, msg: "offset must be either 15 or 31");
87
88 Location loc = op.getLoc();
89 Value res = amdgpu::PermlaneSwapOp::create(
90 builder&: rewriter, location: loc, result: op.getResult(i: 0).getType(), src: op.getValue(), row_length: offsetValue);
91 Value valid = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 1, /*width*/ 1);
92 rewriter.replaceOp(op, newValues: {res, valid});
93 return success();
94 }
95};
96} // namespace
97
98void mlir::populateGpuPromoteShuffleToAMDGPUPatterns(
99 RewritePatternSet &patterns, std::optional<ROCDL::TargetInfo> target) {
100 patterns.add<PromoteShuffleToSwizzlePattern>(arg: patterns.getContext(),
101 /*benefit*/ args: 1);
102 if (target && target->has(feature: llvm::AMDGPU::FEAT_PERMLANE32_SWAP))
103 patterns.add<PromoteShuffleToPermlanePattern>(arg: patterns.getContext(),
104 /*benefit*/ args: 2);
105}
106