1//===- LowerVectorInterleave.cpp - Lower 'vector.interleave' operation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent rewrites and utilities to lower the
10// 'vector.interleave' operation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
16#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
17#include "mlir/IR/BuiltinTypes.h"
18#include "mlir/IR/PatternMatch.h"
19#include "llvm/ADT/SmallVectorExtras.h"
20
21#define DEBUG_TYPE "vector-interleave-lowering"
22
23using namespace mlir;
24using namespace mlir::vector;
25
26namespace {
27
28/// A one-shot unrolling of vector.interleave to the `targetRank`.
29///
30/// Example:
31///
32/// ```mlir
33/// vector.interleave %a, %b : vector<1x2x3x4xi64> -> vector<1x2x3x8xi64>
34/// ```
35/// Would be unrolled to:
36/// ```mlir
37/// %result = arith.constant dense<0> : vector<1x2x3x8xi64>
38/// %0 = vector.extract %a[0, 0, 0] ─┐
39/// : vector<4xi64> from vector<1x2x3x4xi64> |
40/// %1 = vector.extract %b[0, 0, 0] |
41/// : vector<4xi64> from vector<1x2x3x4xi64> | - Repeated 6x for
42/// %2 = vector.interleave %0, %1 : | all leading positions
43/// : vector<4xi64> -> vector<8xi64> |
44/// %3 = vector.insert %2, %result [0, 0, 0] |
45/// : vector<8xi64> into vector<1x2x3x8xi64> ┘
46/// ```
47///
48/// Note: If any leading dimension before the `targetRank` is scalable the
49/// unrolling will stop before the scalable dimension.
50class UnrollInterleaveOp final : public OpRewritePattern<vector::InterleaveOp> {
51public:
52 UnrollInterleaveOp(int64_t targetRank, MLIRContext *context,
53 PatternBenefit benefit = 1)
54 : OpRewritePattern(context, benefit), targetRank(targetRank) {};
55
56 LogicalResult matchAndRewrite(vector::InterleaveOp op,
57 PatternRewriter &rewriter) const override {
58 VectorType resultType = op.getResultVectorType();
59 auto unrollIterator = vector::createUnrollIterator(vType: resultType, targetRank);
60 if (!unrollIterator)
61 return failure();
62
63 auto loc = op.getLoc();
64 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resultType,
65 value: rewriter.getZeroAttr(type: resultType));
66 for (auto position : *unrollIterator) {
67 Value extractLhs =
68 ExtractOp::create(builder&: rewriter, location: loc, source: op.getLhs(), position);
69 Value extractRhs =
70 ExtractOp::create(builder&: rewriter, location: loc, source: op.getRhs(), position);
71 Value interleave =
72 InterleaveOp::create(builder&: rewriter, location: loc, lhs: extractLhs, rhs: extractRhs);
73 result = InsertOp::create(builder&: rewriter, location: loc, valueToStore: interleave, dest: result, position);
74 }
75
76 rewriter.replaceOp(op, newValues: result);
77 return success();
78 }
79
80private:
81 int64_t targetRank = 1;
82};
83
84/// A one-shot unrolling of vector.deinterleave to the `targetRank`.
85///
86/// Example:
87///
88/// ```mlir
89/// %0, %1 = vector.deinterleave %a : vector<1x2x3x8xi64> -> vector<1x2x3x4xi64>
90/// ```
91/// Would be unrolled to:
92/// ```mlir
93/// %result = arith.constant dense<0> : vector<1x2x3x4xi64>
94/// %0 = vector.extract %a[0, 0, 0] ─┐
95/// : vector<8xi64> from vector<1x2x3x8xi64> |
96/// %1, %2 = vector.deinterleave %0 |
97/// : vector<8xi64> -> vector<4xi64> | -- Initial deinterleave
98/// %3 = vector.insert %1, %result [0, 0, 0] | operation unrolled.
99/// : vector<4xi64> into vector<1x2x3x4xi64> |
100/// %4 = vector.insert %2, %result [0, 0, 0] |
101/// : vector<4xi64> into vector<1x2x3x4xi64> ┘
102/// %5 = vector.extract %a[0, 0, 1] ─┐
103/// : vector<8xi64> from vector<1x2x3x8xi64> |
104/// %6, %7 = vector.deinterleave %5 |
105/// : vector<8xi64> -> vector<4xi64> | -- Recursive pattern for
106/// %8 = vector.insert %6, %3 [0, 0, 1] | subsequent unrolled
107/// : vector<4xi64> into vector<1x2x3x4xi64> | deinterleave
108/// %9 = vector.insert %7, %4 [0, 0, 1] | operations. Repeated
109/// : vector<4xi64> into vector<1x2x3x4xi64> ┘ 5x in this case.
110/// ```
111///
112/// Note: If any leading dimension before the `targetRank` is scalable the
113/// unrolling will stop before the scalable dimension.
114class UnrollDeinterleaveOp final
115 : public OpRewritePattern<vector::DeinterleaveOp> {
116public:
117 UnrollDeinterleaveOp(int64_t targetRank, MLIRContext *context,
118 PatternBenefit benefit = 1)
119 : OpRewritePattern(context, benefit), targetRank(targetRank) {};
120
121 LogicalResult matchAndRewrite(vector::DeinterleaveOp op,
122 PatternRewriter &rewriter) const override {
123 VectorType resultType = op.getResultVectorType();
124 auto unrollIterator = vector::createUnrollIterator(vType: resultType, targetRank);
125 if (!unrollIterator)
126 return failure();
127
128 auto loc = op.getLoc();
129 Value emptyResult = arith::ConstantOp::create(
130 builder&: rewriter, location: loc, result: resultType, value: rewriter.getZeroAttr(type: resultType));
131 Value evenResult = emptyResult;
132 Value oddResult = emptyResult;
133
134 for (auto position : *unrollIterator) {
135 auto extractSrc =
136 vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), position);
137 auto deinterleave =
138 vector::DeinterleaveOp::create(builder&: rewriter, location: loc, source: extractSrc);
139 evenResult = vector::InsertOp::create(
140 builder&: rewriter, location: loc, valueToStore: deinterleave.getResultEven(), dest: evenResult, position);
141 oddResult = vector::InsertOp::create(
142 builder&: rewriter, location: loc, valueToStore: deinterleave.getResultOdd(), dest: oddResult, position);
143 }
144 rewriter.replaceOp(op, newValues: ValueRange{evenResult, oddResult});
145 return success();
146 }
147
148private:
149 int64_t targetRank = 1;
150};
151
152/// Rewrite vector.interleave op into an equivalent vector.shuffle op, when
153/// applicable: `sourceType` must be 0D or 1D, and non-scalable.
154///
155/// Example:
156///
157/// ```mlir
158/// vector.interleave %a, %b : vector<7xi16> -> vector<14xi16>
159/// ```
160///
161/// Is rewritten into:
162///
163/// ```mlir
164/// vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13]
165/// : vector<7xi16>, vector<7xi16>
166/// ```
167struct InterleaveToShuffle final : OpRewritePattern<vector::InterleaveOp> {
168 using Base::Base;
169
170 LogicalResult matchAndRewrite(vector::InterleaveOp op,
171 PatternRewriter &rewriter) const override {
172 VectorType sourceType = op.getSourceVectorType();
173 if (sourceType.getRank() > 1 || sourceType.isScalable()) {
174 return failure();
175 }
176 int64_t n = sourceType.getNumElements();
177 auto seq = llvm::seq<int64_t>(Size: 2 * n);
178 auto zip = llvm::map_to_vector(
179 C&: seq, F: [n](int64_t i) { return (i % 2 ? n : 0) + i / 2; });
180 rewriter.replaceOpWithNewOp<ShuffleOp>(op, args: op.getLhs(), args: op.getRhs(), args&: zip);
181 return success();
182 }
183};
184
185/// Rewrite vector.deinterleave op into two equivalent vector.shuffle ops, when
186/// applicable: `sourceType` must be 1D and non-scalable.
187///
188/// Example:
189///
190/// ```mlir
191/// %evens, %odds = vector.deinterleave %arg0 : vector<4xi32> -> vector<2xi32>
192/// ```
193///
194/// Is rewritten into:
195///
196/// ```mlir
197/// %evens = vector.shuffle %arg0, %arg0 [0, 2] : vector<4xi32>, vector<4xi32>
198/// %odds = vector.shuffle %arg0, %arg0 [1, 3] : vector<4xi32>, vector<4xi32>
199/// ```
200struct DeinterleaveToShuffle final : OpRewritePattern<vector::DeinterleaveOp> {
201 using Base::Base;
202
203 LogicalResult matchAndRewrite(vector::DeinterleaveOp op,
204 PatternRewriter &rewriter) const override {
205 VectorType sourceType = op.getSourceVectorType();
206 if (sourceType.getRank() != 1 || sourceType.isScalable()) {
207 return failure();
208 }
209
210 auto seq = llvm::seq<int64_t>(Size: sourceType.getNumElements() / 2);
211 auto evenZip = llvm::map_to_vector(C&: seq, F: [](int64_t i) { return i * 2; });
212 auto oddZip = llvm::map_to_vector(C&: evenZip, F: [](int64_t i) { return i + 1; });
213
214 Value evenResult = vector::ShuffleOp::create(
215 builder&: rewriter, location: op.getLoc(), v1: op.getOperand(), v2: op.getOperand(), mask: evenZip);
216 Value oddResult = vector::ShuffleOp::create(
217 builder&: rewriter, location: op.getLoc(), v1: op.getOperand(), v2: op.getOperand(), mask: oddZip);
218
219 rewriter.replaceOp(op, newValues: ValueRange{evenResult, oddResult});
220 return success();
221 }
222};
223
224} // namespace
225
226void mlir::vector::populateVectorInterleaveLoweringPatterns(
227 RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) {
228 patterns.add<UnrollInterleaveOp, UnrollDeinterleaveOp>(
229 arg&: targetRank, args: patterns.getContext(), args&: benefit);
230}
231
232void mlir::vector::populateVectorInterleaveToShufflePatterns(
233 RewritePatternSet &patterns, PatternBenefit benefit) {
234 patterns.add<InterleaveToShuffle>(arg: patterns.getContext(), args&: benefit);
235}
236
237void mlir::vector::populateVectorDeinterleaveToShufflePatterns(
238 RewritePatternSet &patterns, PatternBenefit benefit) {
239 patterns.add<DeinterleaveToShuffle>(arg: patterns.getContext(), args&: benefit);
240}
241