1//===- Transforms.cpp ---------------------------------------------- C++ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Shard/Transforms/Transforms.h"
10#include "TransformsDetail.h"
11#include "mlir/Dialect/Affine/IR/AffineOps.h"
12#include "mlir/Dialect/Affine/Utils.h"
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Arith/Utils/Utils.h"
15#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
16#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
17#include "mlir/Dialect/Shard/IR/ShardDialect.h"
18#include "mlir/Dialect/Shard/IR/ShardOps.h"
19#include "mlir/Dialect/Tensor/IR/Tensor.h"
20#include "mlir/Dialect/Utils/StaticValueUtils.h"
21#include "mlir/IR/BuiltinTypes.h"
22#include "mlir/IR/DialectRegistry.h"
23#include "mlir/IR/OpDefinition.h"
24#include "mlir/IR/PatternMatch.h"
25#include "mlir/IR/Value.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVector.h"
28#include <iterator>
29#include <numeric>
30
31namespace mlir::shard {
32
33namespace {
34
35/// Lower `shard.process_multi_index` into expression using
36/// `shard.process_linear_index` and `shard.grid_shape`.
37struct ProcessMultiIndexOpLowering
38 : OpRewritePatternWithSymbolTableCollection<ProcessMultiIndexOp> {
39 using OpRewritePatternWithSymbolTableCollection::
40 OpRewritePatternWithSymbolTableCollection;
41
42 LogicalResult matchAndRewrite(ProcessMultiIndexOp op,
43 PatternRewriter &rewriter) const override {
44 GridOp grid = getGrid(op, symbolTableCollection);
45 if (!grid) {
46 return failure();
47 }
48
49 ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
50 builder.setInsertionPointAfter(op.getOperation());
51 Value linearIndex = ProcessLinearIndexOp::create(builder, grid);
52 ValueRange gridShape = GridShapeOp::create(builder, grid).getResults();
53 SmallVector<Value> completeMultiIndex =
54 affine::AffineDelinearizeIndexOp::create(builder, linear_index: linearIndex,
55 basis: gridShape)
56 .getMultiIndex();
57 SmallVector<Value> multiIndex;
58 ArrayRef<GridAxis> opGridAxes = op.getAxes();
59 SmallVector<GridAxis> opAxesIota;
60 if (opGridAxes.empty()) {
61 opAxesIota.resize(N: grid.getRank());
62 std::iota(first: opAxesIota.begin(), last: opAxesIota.end(), value: 0);
63 opGridAxes = opAxesIota;
64 }
65 llvm::transform(Range&: opGridAxes, d_first: std::back_inserter(x&: multiIndex),
66 F: [&completeMultiIndex](GridAxis gridAxis) {
67 return completeMultiIndex[gridAxis];
68 });
69 rewriter.replaceOp(op, newValues: multiIndex);
70 return success();
71 }
72};
73
74struct AllSliceOpLowering
75 : OpRewritePatternWithSymbolTableCollection<AllSliceOp> {
76 using OpRewritePatternWithSymbolTableCollection::
77 OpRewritePatternWithSymbolTableCollection;
78
79 LogicalResult matchAndRewrite(AllSliceOp op,
80 PatternRewriter &rewriter) const override {
81 // 1. Compute the process linear index inside the process group from its
82 // multi-index.
83 //
84 // 2. Extract a slice from the input tensor.
85 // All axes except the slicing axis are not interesting and take the full
86 // axis.
87 // The slice axis is split into equisized parts with count
88 // the number of processes in the collective process group induced by
89 // the grid axes.
90 // The part for each process is determined by the corresponding
91 // linear-index in the process group.
92 //
93 // There are no collectives that require communication.
94 // Each process operates on its local tensor.
95
96 GridOp grid = getGrid(op, symbolTableCollection);
97 if (!grid) {
98 return failure();
99 }
100
101 ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
102 builder.setInsertionPointAfter(op.getOperation());
103
104 Value zero = arith::ConstantOp::create(builder, value: builder.getIndexAttr(value: 0));
105
106 Operation::result_range processInGroupMultiIndex =
107 ProcessMultiIndexOp::create(builder, grid: grid.getSymName(),
108 axes: op.getGridAxes())
109 .getResults();
110
111 Operation::result_range processGroupShape =
112 GridShapeOp::create(builder, grid: grid.getSymName(), axes: op.getGridAxes())
113 .getResult();
114 Value processGroupSize =
115 createCollectiveProcessGroupSize(grid, axes: op.getGridAxes(), builder);
116
117 int64_t sliceAxis = op.getSliceAxis().getSExtValue();
118 Value operandSliceAxisSize =
119 tensor::DimOp::create(builder, source: op.getOperand(), index: sliceAxis);
120 Value operandSliceAxisSizeModProcessGroupSize =
121 arith::RemUIOp::create(builder, lhs: operandSliceAxisSize, rhs: processGroupSize);
122 Value isTargetShapeExactlyDivisible =
123 arith::CmpIOp::create(builder, predicate: arith::CmpIPredicate::eq,
124 lhs: operandSliceAxisSizeModProcessGroupSize, rhs: zero);
125 cf::AssertOp::create(builder, arg: isTargetShapeExactlyDivisible,
126 msg: "Slicing a tensor with axis size that is "
127 "not exactly divisible by the "
128 "grid process group size is not supported.");
129 Value resultSliceAxisSize =
130 arith::DivUIOp::create(builder, lhs: operandSliceAxisSize, rhs: processGroupSize);
131 OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
132 multiIndex: llvm::to_vector_of<OpFoldResult>(Range&: processInGroupMultiIndex),
133 basis: llvm::to_vector_of<OpFoldResult>(Range&: processGroupShape), builder);
134
135 // insert tensor.extract_slice
136 RankedTensorType operandType =
137 cast<RankedTensorType>(Val: op.getOperand().getType());
138 SmallVector<OpFoldResult> sizes;
139 for (int64_t i = 0; i < operandType.getRank(); ++i) {
140 if (i == sliceAxis) {
141 sizes.emplace_back(Args&: resultSliceAxisSize);
142 } else {
143 Value dimSize = tensor::DimOp::create(builder, source: op.getOperand(), index: i);
144 sizes.emplace_back(Args&: dimSize);
145 }
146 }
147 SmallVector<OpFoldResult> offsets(
148 operandType.getRank(), getAsIndexOpFoldResult(ctx: builder.getContext(), val: 0));
149 offsets[sliceAxis] =
150 ArithBuilder(builder, builder.getLoc())
151 .mul(lhs: getValueOrCreateConstantIndexOp(b&: builder, loc: builder.getLoc(),
152 ofr: processInGroupLinearIndex),
153 rhs: resultSliceAxisSize);
154 SmallVector<OpFoldResult> strides(
155 operandType.getRank(), getAsIndexOpFoldResult(ctx: builder.getContext(), val: 1));
156 Value slice = tensor::ExtractSliceOp::create(builder, source: op.getOperand(),
157 offsets, sizes, strides);
158 Value newResult =
159 tensor::CastOp::create(builder, dest: op.getResult().getType(), source: slice);
160 rewriter.replaceOp(op, newValues: newResult);
161 return success();
162 }
163};
164
165} // namespace
166
167void populateProcessMultiIndexOpLoweringPatterns(
168 RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) {
169 patterns.add<ProcessMultiIndexOpLowering>(arg&: symbolTableCollection,
170 args: patterns.getContext());
171}
172
173void registerProcessMultiIndexOpLoweringDialects(DialectRegistry &registry) {
174 registry.insert<affine::AffineDialect, shard::ShardDialect>();
175}
176
177void populateAllSliceOpLoweringPatterns(
178 RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) {
179 patterns.add<AllSliceOpLowering>(arg&: symbolTableCollection,
180 args: patterns.getContext());
181}
182
183void registerAllSliceOpLoweringDialects(DialectRegistry &registry) {
184 registry.insert<affine::AffineDialect, arith::ArithDialect,
185 cf::ControlFlowDialect, shard::ShardDialect,
186 tensor::TensorDialect>();
187}
188
189void populateAllOpLoweringPatterns(
190 RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) {
191 populateProcessMultiIndexOpLoweringPatterns(patterns, symbolTableCollection);
192 populateAllSliceOpLoweringPatterns(patterns, symbolTableCollection);
193}
194
195void registerAllOpLoweringDialects(DialectRegistry &registry) {
196 registerProcessMultiIndexOpLoweringDialects(registry);
197 registerAllSliceOpLoweringDialects(registry);
198}
199
200TypedValue<IndexType>
201createCollectiveProcessGroupSize(GridOp grid, ArrayRef<GridAxis> axes,
202 ImplicitLocOpBuilder &builder) {
203 Operation::result_range gridShape =
204 GridShapeOp::create(builder, grid, axes).getResults();
205 return cast<TypedValue<IndexType>>(Val: arith::createProduct(
206 builder, loc: builder.getLoc(), values: llvm::to_vector_of<Value>(Range&: gridShape),
207 resultType: builder.getIndexType()));
208}
209
210TypedValue<IndexType>
211createProcessLinearIndex(ImplicitLocOpBuilder &builder, StringRef grid,
212 ValueRange processInGroupMultiIndex,
213 ArrayRef<GridAxis> gridAxes) {
214 Operation::result_range processGroupShape =
215 GridShapeOp::create(builder, grid, axes: gridAxes).getResult();
216 OpFoldResult processInGroupLinearIndex = affine::linearizeIndex(
217 multiIndex: llvm::to_vector_of<OpFoldResult>(Range&: processInGroupMultiIndex),
218 basis: llvm::to_vector_of<OpFoldResult>(Range&: processGroupShape), builder);
219 auto res = dyn_cast<Value>(Val&: processInGroupLinearIndex);
220 if (!res)
221 res = arith::ConstantIndexOp::create(
222 builder,
223 value: cast<IntegerAttr>(Val: cast<Attribute>(Val&: processInGroupLinearIndex)).getInt());
224 return cast<TypedValue<IndexType>>(Val&: res);
225}
226
227TypedValue<IndexType> createProcessLinearIndex(ImplicitLocOpBuilder &builder,
228 StringRef grid,
229 ArrayRef<GridAxis> gridAxes) {
230 return createProcessLinearIndex(
231 builder, grid,
232 processInGroupMultiIndex: ProcessMultiIndexOp::create(builder, grid, axes: gridAxes).getResults(),
233 gridAxes);
234}
235} // namespace mlir::shard
236