1//===- LowerVectorBroadcast.cpp - Lower 'vector.broadcast' operation ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent rewrites and utilities to lower the
10// 'vector.broadcast' operation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/MemRef/IR/MemRef.h"
15#include "mlir/Dialect/UB/IR/UBOps.h"
16#include "mlir/Dialect/Vector/IR/VectorOps.h"
17#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
18#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
19#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
20#include "mlir/IR/BuiltinTypes.h"
21#include "mlir/IR/Location.h"
22#include "mlir/IR/PatternMatch.h"
23#include "mlir/IR/TypeUtilities.h"
24
25#define DEBUG_TYPE "vector-broadcast-lowering"
26
27using namespace mlir;
28using namespace mlir::vector;
29
30namespace {
31
32/// Convert a vector.broadcast with a vector operand to a lower rank
33/// vector.broadcast. vector.broadcast with a scalar operand is expected to be
34/// convertible to the lower level target dialect (LLVM, SPIR-V, etc.) directly.
35class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> {
36public:
37 using Base::Base;
38
39 LogicalResult matchAndRewrite(vector::BroadcastOp op,
40 PatternRewriter &rewriter) const override {
41 auto loc = op.getLoc();
42 VectorType dstType = op.getResultVectorType();
43 VectorType srcType = dyn_cast<VectorType>(Val: op.getSourceType());
44 Type eltType = dstType.getElementType();
45
46 // A broadcast from a scalar is considered to be in the lowered form.
47 if (!srcType)
48 return rewriter.notifyMatchFailure(
49 arg&: op, msg: "broadcast from scalar already in lowered form");
50
51 // Determine rank of source and destination.
52 int64_t srcRank = srcType.getRank();
53 int64_t dstRank = dstType.getRank();
54
55 // Single-element fixed-size source: extract the scalar and broadcast it.
56 if (srcType.getNumElements() == 1 && !srcType.isScalable()) {
57 SmallVector<int64_t> fullRankPosition(srcRank, 0);
58 Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(),
59 position: fullRankPosition);
60 assert(!isa<VectorType>(ext.getType()) && "expected scalar");
61 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, args&: dstType, args&: ext);
62 return success();
63 }
64
65 // Duplicate this rank.
66 // For example:
67 // %x = broadcast %y : k-D to n-D, k < n
68 // becomes:
69 // %b = broadcast %y : k-D to (n-1)-D
70 // %x = [%b,%b,%b,%b] : n-D
71 // becomes:
72 // %b = [%y,%y] : (n-1)-D
73 // %x = [%b,%b,%b,%b] : n-D
74 if (srcRank < dstRank) {
75 // Duplication.
76 if (dstType.getScalableDims()[0])
77 return rewriter.notifyMatchFailure(
78 arg&: op, msg: "Vector broadcasting over a scalable dimension is not "
79 "currently supported");
80 VectorType resType = VectorType::Builder(dstType).dropDim(pos: 0);
81 Value bcst =
82 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: op.getSource());
83 Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: dstType);
84 for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d)
85 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d);
86 rewriter.replaceOp(op, newValues: result);
87 return success();
88 }
89
90 // Find non-matching dimension, if any.
91 assert(srcRank == dstRank);
92 int64_t m = -1;
93 for (int64_t r = 0; r < dstRank; r++)
94 if (srcType.getDimSize(idx: r) != dstType.getDimSize(idx: r)) {
95 m = r;
96 break;
97 }
98
99 // All trailing dimensions are the same. Simply pass through.
100 if (m == -1) {
101 rewriter.replaceOp(op, newValues: op.getSource());
102 return success();
103 }
104
105 // Any non-matching dimension forces a stretch along this rank.
106 // For example:
107 // %x = broadcast %y : vector<4x1x2xf32> to vector<4x2x2xf32>
108 // becomes:
109 // %a = broadcast %y[0] : vector<1x2xf32> to vector<2x2xf32>
110 // %b = broadcast %y[1] : vector<1x2xf32> to vector<2x2xf32>
111 // %c = broadcast %y[2] : vector<1x2xf32> to vector<2x2xf32>
112 // %d = broadcast %y[3] : vector<1x2xf32> to vector<2x2xf32>
113 // %x = [%a,%b,%c,%d]
114 // becomes:
115 // %u = broadcast %y[0][0] : vector<2xf32> to vector <2x2xf32>
116 // %v = broadcast %y[1][0] : vector<2xf32> to vector <2x2xf32>
117 // %a = [%u, %v]
118 // ..
119 // %x = [%a,%b,%c,%d]
120 VectorType resType =
121 VectorType::get(shape: dstType.getShape().drop_front(), elementType: eltType,
122 scalableDims: dstType.getScalableDims().drop_front());
123
124 // For a scalable outer dimension we would need to emit an scf.for loop,
125 // which is not yet supported. Check before creating any IR so that
126 // returning failure() does not violate the pattern API contract.
127 if (dstType.getScalableDims()[0]) {
128 // TODO: For scalable vectors we should emit an scf.for loop.
129 return failure();
130 }
131
132 Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: dstType);
133 if (m == 0) {
134 // Stetch at start.
135 Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), position: 0);
136 Value bcst = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: ext);
137 for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d)
138 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d);
139 } else {
140 // Stetch not at start.
141 for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d) {
142 Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), position: d);
143 Value bcst = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: ext);
144 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d);
145 }
146 }
147 rewriter.replaceOp(op, newValues: result);
148 return success();
149 }
150};
151} // namespace
152
153void mlir::vector::populateVectorBroadcastLoweringPatterns(
154 RewritePatternSet &patterns, PatternBenefit benefit) {
155 patterns.add<BroadcastOpLowering>(arg: patterns.getContext(), args&: benefit);
156}
157