| 1 | //===- ShuffleRewriter.cpp - Implementation of shuffle rewriting ---------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements in-dialect rewriting of the shuffle op for types i64 and |
| 10 | // f64, rewriting 64bit shuffles into two 32bit shuffles. This particular |
| 11 | // implementation using shifts and truncations can be obtained using clang: by |
| 12 | // emitting IR for shuffle operations with `-O3`. |
| 13 | // |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
| 16 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 17 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 18 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 19 | #include "mlir/IR/Builders.h" |
| 20 | #include "mlir/IR/PatternMatch.h" |
| 21 | |
| 22 | using namespace mlir; |
| 23 | |
| 24 | namespace { |
| 25 | struct GpuShuffleRewriter : public OpRewritePattern<gpu::ShuffleOp> { |
| 26 | using OpRewritePattern<gpu::ShuffleOp>::OpRewritePattern; |
| 27 | |
| 28 | void initialize() { |
| 29 | // Required as the pattern will replace the Op with 2 additional ShuffleOps. |
| 30 | setHasBoundedRewriteRecursion(); |
| 31 | } |
| 32 | LogicalResult matchAndRewrite(gpu::ShuffleOp op, |
| 33 | PatternRewriter &rewriter) const override { |
| 34 | auto loc = op.getLoc(); |
| 35 | auto value = op.getValue(); |
| 36 | auto valueType = value.getType(); |
| 37 | auto valueLoc = value.getLoc(); |
| 38 | auto i32 = rewriter.getI32Type(); |
| 39 | auto i64 = rewriter.getI64Type(); |
| 40 | |
| 41 | // If the type of the value is either i32 or f32, the op is already valid. |
| 42 | if (!valueType.isIntOrFloat() || valueType.getIntOrFloatBitWidth() != 64) |
| 43 | return rewriter.notifyMatchFailure( |
| 44 | arg&: op, msg: "only 64-bit int/float types are supported" ); |
| 45 | |
| 46 | Value lo, hi; |
| 47 | |
| 48 | // Float types must be converted to i64 to extract the bits. |
| 49 | if (isa<FloatType>(Val: valueType)) |
| 50 | value = arith::BitcastOp::create(builder&: rewriter, location: valueLoc, out: i64, in: value); |
| 51 | |
| 52 | // Get the low bits by trunc(value). |
| 53 | lo = arith::TruncIOp::create(builder&: rewriter, location: valueLoc, out: i32, in: value); |
| 54 | |
| 55 | // Get the high bits by trunc(value >> 32). |
| 56 | auto c32 = arith::ConstantOp::create(builder&: rewriter, location: valueLoc, |
| 57 | value: rewriter.getIntegerAttr(type: i64, value: 32)); |
| 58 | hi = arith::ShRUIOp::create(builder&: rewriter, location: valueLoc, lhs: value, rhs: c32); |
| 59 | hi = arith::TruncIOp::create(builder&: rewriter, location: valueLoc, out: i32, in: hi); |
| 60 | |
| 61 | // Shuffle the values. |
| 62 | ValueRange loRes = |
| 63 | gpu::ShuffleOp::create(builder&: rewriter, location: op.getLoc(), value: lo, offset: op.getOffset(), |
| 64 | width: op.getWidth(), mode: op.getMode()) |
| 65 | .getResults(); |
| 66 | ValueRange hiRes = |
| 67 | gpu::ShuffleOp::create(builder&: rewriter, location: op.getLoc(), value: hi, offset: op.getOffset(), |
| 68 | width: op.getWidth(), mode: op.getMode()) |
| 69 | .getResults(); |
| 70 | |
| 71 | // Convert lo back to i64. |
| 72 | lo = arith::ExtUIOp::create(builder&: rewriter, location: valueLoc, out: i64, in: loRes[0]); |
| 73 | |
| 74 | // Convert hi back to i64. |
| 75 | hi = arith::ExtUIOp::create(builder&: rewriter, location: valueLoc, out: i64, in: hiRes[0]); |
| 76 | hi = arith::ShLIOp::create(builder&: rewriter, location: valueLoc, lhs: hi, rhs: c32); |
| 77 | |
| 78 | // Obtain the shuffled bits hi | lo. |
| 79 | value = arith::OrIOp::create(builder&: rewriter, location: loc, lhs: hi, rhs: lo); |
| 80 | |
| 81 | // Convert the value back to float. |
| 82 | if (isa<FloatType>(Val: valueType)) |
| 83 | value = arith::BitcastOp::create(builder&: rewriter, location: valueLoc, out: valueType, in: value); |
| 84 | |
| 85 | // Obtain the shuffle validity by combining both validities. |
| 86 | auto validity = arith::AndIOp::create(builder&: rewriter, location: loc, lhs: loRes[1], rhs: hiRes[1]); |
| 87 | |
| 88 | // Replace the op. |
| 89 | rewriter.replaceOp(op, newValues: {value, validity}); |
| 90 | return success(); |
| 91 | } |
| 92 | }; |
| 93 | } // namespace |
| 94 | |
| 95 | void mlir::populateGpuShufflePatterns(RewritePatternSet &patterns) { |
| 96 | patterns.add<GpuShuffleRewriter>(arg: patterns.getContext()); |
| 97 | } |
| 98 | |