1//===- ShardOps.cpp - Shard Dialect Operations ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Shard/IR/ShardOps.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Shard/IR/ShardDialect.h"
13#include "mlir/Dialect/Utils/StaticValueUtils.h"
14#include "mlir/IR/Attributes.h"
15#include "mlir/IR/BuiltinAttributes.h"
16#include "mlir/IR/BuiltinTypeInterfaces.h"
17#include "mlir/IR/BuiltinTypes.h"
18#include "mlir/IR/Diagnostics.h"
19#include "mlir/IR/IRMapping.h"
20#include "mlir/IR/Location.h"
21#include "mlir/IR/PatternMatch.h"
22#include "mlir/IR/TypeUtilities.h"
23#include "mlir/IR/Value.h"
24#include "mlir/Interfaces/ViewLikeInterface.h"
25#include "mlir/Support/LLVM.h"
26#include "mlir/Transforms/InliningUtils.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include <algorithm>
32#include <functional>
33#include <iterator>
34#include <numeric>
35#include <optional>
36#include <utility>
37
38#define DEBUG_TYPE "shard-ops"
39
40using namespace mlir;
41using namespace mlir::shard;
42
43#include "mlir/Dialect/Shard/IR/ShardDialect.cpp.inc"
44
45namespace {
46
47struct DimensionSize {
48 static DimensionSize dynamic() { return DimensionSize(ShapedType::kDynamic); }
49 DimensionSize(int64_t val) : val(val) {}
50 int64_t value() const { return val; }
51 operator int64_t() const { return val; }
52 bool isDynamic() const { return ShapedType::isDynamic(dValue: val); }
53
54private:
55 int64_t val;
56};
57
58} // namespace
59
60static DimensionSize operator/(DimensionSize lhs, DimensionSize rhs) {
61 if (lhs.isDynamic() || rhs.isDynamic()) {
62 return DimensionSize::dynamic();
63 }
64 return lhs.value() / rhs.value();
65}
66
67static DimensionSize operator*(DimensionSize lhs, DimensionSize rhs) {
68 if (lhs.isDynamic() || rhs.isDynamic()) {
69 return DimensionSize::dynamic();
70 }
71 return lhs.value() * rhs.value();
72}
73
74SmallVector<Value>
75mlir::shard::getMixedAsValues(OpBuilder b, const Location &loc,
76 llvm::ArrayRef<int64_t> statics,
77 ValueRange dynamics, Type type) {
78 SmallVector<Value> values;
79 auto dyn = dynamics.begin();
80 Type i64 = b.getI64Type();
81 if (!type)
82 type = i64;
83 assert((i64 == type || b.getIndexType() == type) &&
84 "expected an i64 or an intex type");
85 for (auto s : statics) {
86 if (s == ShapedType::kDynamic) {
87 values.emplace_back(Args: *(dyn++));
88 } else {
89 TypedAttr val = type == i64 ? b.getI64IntegerAttr(value: s) : b.getIndexAttr(value: s);
90 values.emplace_back(Args: arith::ConstantOp::create(builder&: b, location: loc, result: type, value: val));
91 }
92 }
93 return values;
94}
95
96//===----------------------------------------------------------------------===//
97// Inliner
98//===----------------------------------------------------------------------===//
99
100namespace {
101struct ShardInlinerinterface : public DialectInlinerInterface {
102 using DialectInlinerInterface::DialectInlinerInterface;
103 // Currently no restrictions are encoded for inlining.
104 bool isLegalToInline(Operation *, Operation *, bool) const final {
105 return true;
106 }
107 bool isLegalToInline(Region *, Region *, bool, IRMapping &) const final {
108 return true;
109 }
110 bool isLegalToInline(Operation *, Region *, bool, IRMapping &) const final {
111 return true;
112 }
113};
114} // namespace
115
116//===----------------------------------------------------------------------===//
117// Shard dialect
118//===----------------------------------------------------------------------===//
119
120void ShardDialect::initialize() {
121 addOperations<
122#define GET_OP_LIST
123#include "mlir/Dialect/Shard/IR/ShardOps.cpp.inc"
124 >();
125 addAttributes<
126#define GET_ATTRDEF_LIST
127#include "mlir/Dialect/Shard/IR/ShardAttributes.cpp.inc"
128 >();
129 addTypes<
130#define GET_TYPEDEF_LIST
131#include "mlir/Dialect/Shard/IR/ShardTypes.cpp.inc"
132 >();
133 addInterface<ShardInlinerinterface>();
134}
135
136Operation *ShardDialect::materializeConstant(OpBuilder &builder,
137 Attribute value, Type type,
138 Location loc) {
139 return arith::ConstantOp::materialize(builder, value, type, loc);
140}
141
142//===----------------------------------------------------------------------===//
143// Shard utilities
144//===----------------------------------------------------------------------===//
145
146static FailureOr<GridOp> getGridAndVerify(Operation *op,
147 FlatSymbolRefAttr gridSymbol,
148 SymbolTableCollection &symbolTable) {
149 shard::GridOp grid = getGridOrNull(op, gridSymbol, symbolTableCollection&: symbolTable);
150 if (!grid) {
151 return op->emitError() << "Undefined required grid symbol \""
152 << gridSymbol.getValue() << "\".";
153 }
154
155 return grid;
156}
157
158template <typename It>
159static bool isUnique(It begin, It end) {
160 if (begin == end) {
161 return true;
162 }
163 It next = std::next(begin);
164 if (next == end) {
165 return true;
166 }
167 for (; next != end; ++next, ++begin) {
168 if (*begin == *next) {
169 return false;
170 }
171 }
172 return true;
173}
174
175static LogicalResult verifyGridAxes(Location loc, ArrayRef<GridAxis> axes,
176 GridOp grid) {
177 SmallVector<GridAxis> sorted = llvm::to_vector(Range&: axes);
178 llvm::sort(C&: sorted);
179 if (!isUnique(begin: sorted.begin(), end: sorted.end())) {
180 return emitError(loc) << "Grid axes contains duplicate elements.";
181 }
182
183 GridAxis rank = grid.getRank();
184 for (auto axis : axes) {
185 if (axis >= rank || axis < 0) {
186 return emitError(loc)
187 << "0-based grid axis index " << axis
188 << " is out of bounds. The referenced grid \"" << grid.getSymName()
189 << "\" is of rank " << rank << ".";
190 }
191 }
192
193 return success();
194}
195
196template <typename Op>
197static FailureOr<GridOp>
198getGridAndVerifyAxes(Op op, SymbolTableCollection &symbolTable) {
199 auto grid =
200 ::getGridAndVerify(op: op.getOperation(), gridSymbol: op.getGridAttr(), symbolTable);
201 if (failed(grid)) {
202 return failure();
203 }
204 if (failed(verifyGridAxes(op.getLoc(), op.getGridAxes(), grid.value()))) {
205 return failure();
206 }
207 return grid;
208}
209
210template <typename InShape, typename GridShape, typename SplitAxes,
211 typename OutShape>
212static void shardShape(const InShape &inShape, const GridShape &gridShape,
213 const SplitAxes &splitAxes, OutShape &outShape,
214 ArrayRef<int64_t> shardedDimsOffsets = {},
215 ArrayRef<int64_t> haloSizes = {}) {
216 // 0d tensors cannot be sharded and must get replicated
217 if (inShape.empty()) {
218 assert(outShape.empty());
219 return;
220 }
221
222 std::copy(llvm::adl_begin(inShape), llvm::adl_end(inShape),
223 llvm::adl_begin(outShape));
224
225 if (!shardedDimsOffsets.empty()) {
226 auto isDynShape = ShapedType::isDynamicShape(dSizes: gridShape);
227 uint64_t pos = 1;
228 for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(splitAxes)) {
229 if (!innerSplitAxes.empty()) {
230 auto sz = shardedDimsOffsets[pos];
231 bool same = !isDynShape;
232 if (same) {
233 // Find sharded dims in shardedDimsOffsets with same static size on
234 // all devices. Use kDynamic for dimensions with dynamic or
235 // non-uniform offs in shardedDimsOffsets.
236 uint64_t numShards = 0;
237 for (auto i : innerSplitAxes.asArrayRef()) {
238 numShards += gridShape[i];
239 }
240 for (size_t i = 1; i < numShards; ++i) {
241 if (shardedDimsOffsets[pos + i] - shardedDimsOffsets[pos + i - 1] !=
242 sz) {
243 same = false;
244 break;
245 }
246 }
247 pos += numShards + 1;
248 }
249 outShape[tensorAxis] = same ? sz : ShapedType::kDynamic;
250 }
251 }
252 } else {
253 for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(splitAxes)) {
254 outShape[tensorAxis] = shardDimension(
255 inShape[tensorAxis],
256 collectiveProcessGroupSize(innerSplitAxes.asArrayRef(), gridShape));
257 }
258
259 if (!haloSizes.empty()) {
260 // add halo sizes if requested
261 int haloAxis = 0;
262 for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(splitAxes)) {
263 if (ShapedType::isStatic(dValue: outShape[tensorAxis]) &&
264 !innerSplitAxes.empty()) {
265 if (haloSizes[haloAxis * 2] >= 0 &&
266 haloSizes[haloAxis * 2 + 1] >= 0) {
267 outShape[tensorAxis] +=
268 haloSizes[haloAxis * 2] + haloSizes[haloAxis * 2 + 1];
269 ++haloAxis;
270 } else {
271 outShape[tensorAxis] = ShapedType::kDynamic;
272 }
273 }
274 }
275 }
276 }
277}
278
279ShapedType shard::shardShapedType(ShapedType shape, GridOp grid,
280 Sharding sharding) {
281 using Dim = std::decay_t<decltype(shape.getDimSize(idx: 0))>;
282 SmallVector<Dim> resShapeArr(shape.getShape().size());
283 shardShape(inShape: shape.getShape(), gridShape: grid.getShape(), splitAxes: sharding.getSplitAxes(),
284 outShape&: resShapeArr, shardedDimsOffsets: sharding.getStaticShardedDimsOffsets(),
285 haloSizes: sharding.getStaticHaloSizes());
286 return shape.clone(shape: resShapeArr);
287}
288
289Type shard::shardType(Type type, GridOp grid, Sharding sharding) {
290 RankedTensorType rankedTensorType = dyn_cast<RankedTensorType>(Val&: type);
291 if (rankedTensorType && !rankedTensorType.getShape().empty()) {
292 return shardShapedType(shape: rankedTensorType, grid, sharding);
293 }
294 return type;
295}
296
297static void maybeInsertTargetShardingAnnotationImpl(Sharding sharding,
298 Value &operandValue,
299 Operation *operandOp,
300 OpBuilder &builder,
301 ShardOp &newShardOp) {
302 OpBuilder::InsertionGuard insertionGuard(builder);
303 builder.setInsertionPointAfterValue(operandValue);
304 ShardOp shardOp = dyn_cast<ShardOp>(Val: operandOp);
305 if (shardOp && sharding == shardOp.getSharding() &&
306 !shardOp.getAnnotateForUsers()) {
307 // No need for anything if the correct sharding is already set.
308 if (!newShardOp) {
309 newShardOp = shardOp;
310 }
311 return;
312 }
313
314 if (!newShardOp) {
315 auto shardingOp =
316 ShardingOp::create(builder, location: operandValue.getLoc(), from: sharding);
317 newShardOp = ShardOp::create(builder, location: operandValue.getLoc(), src: operandValue,
318 sharding: shardingOp,
319 /*annotate_for_users*/ false);
320 }
321 operandValue.replaceUsesWithIf(
322 newValue: newShardOp, shouldReplace: [operandOp, operandValue](OpOperand &use) {
323 return use.getOwner() == operandOp && use.get() == operandValue;
324 });
325
326 if (!shardOp || shardOp.getAnnotateForUsers()) {
327 return;
328 }
329
330 auto newShardOp2 = ShardOp::create(builder, location: operandValue.getLoc(), src: newShardOp,
331 sharding: newShardOp.getSharding(),
332 /*annotate_for_users*/ true);
333 newShardOp.getResult().replaceAllUsesExcept(newValue: newShardOp2, exceptedUser: newShardOp2);
334}
335
336void mlir::shard::maybeInsertTargetShardingAnnotation(Sharding sharding,
337 OpResult result,
338 OpBuilder &builder) {
339 ShardOp newShardOp;
340 SmallVector<std::pair<Value, Operation *>> uses;
341 for (auto &use : result.getUses()) {
342 uses.emplace_back(Args: use.get(), Args: use.getOwner());
343 }
344 for (auto &[operandValue, operandOp] : uses) {
345 maybeInsertTargetShardingAnnotationImpl(sharding, operandValue, operandOp,
346 builder, newShardOp);
347 }
348}
349
350void mlir::shard::maybeInsertSourceShardingAnnotation(Sharding sharding,
351 OpOperand &operand,
352 OpBuilder &builder) {
353 OpBuilder::InsertionGuard insertionGuard(builder);
354 Value operandValue = operand.get();
355 Operation *operandSrcOp = operandValue.getDefiningOp();
356 bool isBlockArg = !operandSrcOp;
357 {
358 [[maybe_unused]] auto opType =
359 dyn_cast<mlir::RankedTensorType>(Val: operandValue.getType());
360 assert(!opType || opType.getRank() > 0 || isFullReplication(sharding));
361 }
362 if (!isa<RankedTensorType>(Val: operandValue.getType()) && operandSrcOp &&
363 operandSrcOp->hasTrait<OpTrait::ConstantLike>()) {
364 return;
365 }
366
367 Operation *operandOp = operand.getOwner();
368 ShardOp shardOp = dyn_cast_or_null<ShardOp>(Val: operandSrcOp);
369
370 if (shardOp && sharding == shardOp.getSharding() &&
371 shardOp.getAnnotateForUsers()) {
372 // No need for anything the correct sharding is already set.
373 return;
374 }
375
376 builder.setInsertionPoint(operandOp);
377 auto shardingOp =
378 ShardingOp::create(builder, location: operand.get().getLoc(), from: sharding);
379 auto newShardOp =
380 ShardOp::create(builder, location: operandValue.getLoc(), src: operandValue, sharding: shardingOp,
381 /*annotate_for_users*/ true);
382 IRRewriter rewriter(builder);
383 rewriter.replaceUsesWithIf(
384 from: operandValue, to: newShardOp, functor: [operandOp, operandValue](OpOperand &use) {
385 return use.getOwner() == operandOp && use.get() == operandValue;
386 });
387
388 if (isBlockArg || !shardOp || !shardOp.getAnnotateForUsers()) {
389 // No need for resharding.
390 return;
391 }
392
393 builder.setInsertionPoint(newShardOp);
394 auto newPreceedingShardOp =
395 ShardOp::create(builder, location: operandValue.getLoc(), src: operandValue, sharding: shardingOp,
396 /*annotate_for_users*/ false);
397 rewriter.replaceUsesWithIf(
398 from: newShardOp.getSrc(), to: newPreceedingShardOp, functor: [&newShardOp](OpOperand &use) {
399 return use.getOwner() == newShardOp.getOperation();
400 });
401}
402
403//===----------------------------------------------------------------------===//
404// shard.grid op
405//===----------------------------------------------------------------------===//
406
407LogicalResult GridOp::verify() {
408 int64_t rank = getRank();
409
410 if (rank <= 0)
411 return emitOpError(message: "rank of grid is expected to be a positive integer");
412
413 for (int64_t dimSize : getShape()) {
414 if (dimSize < 0 && ShapedType::isStatic(dValue: dimSize))
415 return emitOpError(message: "dimension size of a grid is expected to be "
416 "non-negative or dynamic");
417 }
418
419 return success();
420}
421
422//===----------------------------------------------------------------------===//
423// shard.grid_shape op
424//===----------------------------------------------------------------------===//
425
426LogicalResult
427GridShapeOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
428 auto grid = ::getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
429 if (failed(Result: grid)) {
430 return failure();
431 }
432 if (failed(Result: verifyGridAxes(loc: getLoc(), axes: getAxes(), grid: grid.value()))) {
433 return failure();
434 }
435
436 size_t expectedResultsCount =
437 getAxes().empty() ? grid->getRank() : getAxes().size();
438 if (getResult().size() != expectedResultsCount) {
439 return emitError() << "Unexpected number of results " << getResult().size()
440 << ". Expected " << expectedResultsCount << ".";
441 }
442
443 return success();
444}
445
446void GridShapeOp::build(OpBuilder &odsBuilder, OperationState &odsState,
447 GridOp grid) {
448 build(odsBuilder, odsState, grid, axes: SmallVector<GridAxis>());
449}
450
451void GridShapeOp::build(OpBuilder &odsBuilder, OperationState &odsState,
452 GridOp grid, ArrayRef<GridAxis> axes) {
453 build(odsBuilder, odsState,
454 result: SmallVector<Type>(axes.empty() ? grid.getRank() : axes.size(),
455 odsBuilder.getIndexType()),
456 grid: grid.getSymName(), axes: GridAxesAttr::get(context: odsBuilder.getContext(), content: axes));
457}
458
459void GridShapeOp::build(OpBuilder &odsBuilder, OperationState &odsState,
460 StringRef grid, ArrayRef<GridAxis> axes) {
461 assert(!axes.empty());
462 build(odsBuilder, odsState,
463 result: SmallVector<Type>(axes.size(), odsBuilder.getIndexType()), grid,
464 axes: GridAxesAttr::get(context: odsBuilder.getContext(), content: axes));
465}
466
467void GridShapeOp::getAsmResultNames(
468 function_ref<void(Value, StringRef)> setNameFn) {
469 setNameFn(getResults()[0], "grid_shape");
470}
471
472//===----------------------------------------------------------------------===//
473// shard.sharding
474//===----------------------------------------------------------------------===//
475
476void ShardingOp::build(::mlir::OpBuilder &b, ::mlir::OperationState &odsState,
477 FlatSymbolRefAttr grid, ArrayRef<GridAxesAttr> splitAxes,
478 ArrayRef<int64_t> staticHalos,
479 ArrayRef<int64_t> staticOffsets) {
480 return build(
481 odsBuilder&: b, odsState, grid, split_axes: GridAxesArrayAttr::get(context: b.getContext(), axes: splitAxes),
482 static_sharded_dims_offsets: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticHalos), dynamic_sharded_dims_offsets: {},
483 static_halo_sizes: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticOffsets), dynamic_halo_sizes: {});
484}
485
486void ShardingOp::build(::mlir::OpBuilder &b, ::mlir::OperationState &odsState,
487 llvm::StringRef grid, ArrayRef<GridAxesAttr> splitAxes,
488 ArrayRef<int64_t> staticHalos,
489 ArrayRef<int64_t> staticOffsets) {
490 return build(odsBuilder&: b, odsState, grid: FlatSymbolRefAttr::get(ctx: b.getContext(), value: grid),
491 split_axes: GridAxesArrayAttr::get(context: b.getContext(), axes: splitAxes),
492 static_sharded_dims_offsets: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticHalos), dynamic_sharded_dims_offsets: {},
493 static_halo_sizes: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticOffsets),
494 dynamic_halo_sizes: {});
495}
496
497void ShardingOp::build(
498 ::mlir::OpBuilder &b, ::mlir::OperationState &odsState,
499 FlatSymbolRefAttr grid, ArrayRef<GridAxesAttr> splitAxes,
500 ::mlir::ArrayRef<::mlir::OpFoldResult> haloSizes,
501 ::mlir::ArrayRef<::mlir::OpFoldResult> shardedDimsOffsets) {
502 mlir::SmallVector<int64_t> staticHalos, staticDims;
503 mlir::SmallVector<mlir::Value> dynamicHalos, dynamicDims;
504 dispatchIndexOpFoldResults(ofrs: haloSizes, dynamicVec&: dynamicHalos, staticVec&: staticHalos);
505 dispatchIndexOpFoldResults(ofrs: shardedDimsOffsets, dynamicVec&: dynamicDims, staticVec&: staticDims);
506 return build(
507 odsBuilder&: b, odsState, grid, split_axes: GridAxesArrayAttr::get(context: b.getContext(), axes: splitAxes),
508 static_sharded_dims_offsets: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticHalos), dynamic_sharded_dims_offsets: dynamicHalos,
509 static_halo_sizes: ::mlir::DenseI64ArrayAttr::get(context: b.getContext(), content: staticDims), dynamic_halo_sizes: dynamicDims);
510}
511
512void ShardingOp::build(::mlir::OpBuilder &b, ::mlir::OperationState &odsState,
513 mlir::shard::Sharding from) {
514
515 build(odsBuilder&: b, odsState, result: ShardingType::get(ctx: b.getContext()), grid: from.getGridAttr(),
516 split_axes: GridAxesArrayAttr::get(context: b.getContext(), axes: from.getSplitAxes()),
517 static_sharded_dims_offsets: from.getStaticShardedDimsOffsets().empty()
518 ? DenseI64ArrayAttr()
519 : b.getDenseI64ArrayAttr(values: from.getStaticShardedDimsOffsets()),
520 dynamic_sharded_dims_offsets: from.getDynamicShardedDimsOffsets(),
521 static_halo_sizes: from.getStaticHaloSizes().empty()
522 ? DenseI64ArrayAttr()
523 : b.getDenseI64ArrayAttr(values: from.getStaticHaloSizes()),
524 dynamic_halo_sizes: from.getDynamicHaloSizes());
525}
526
527LogicalResult ShardingOp::verify() {
528 llvm::SmallSet<GridAxis, 4> visitedAxes;
529
530 auto checkGridAxis = [&](ArrayRef<GridAxis> axesArray) -> LogicalResult {
531 for (GridAxis axis : axesArray) {
532 if (axis < 0)
533 return emitError() << "grid axis is expected to be non-negative";
534 if (!visitedAxes.insert(V: axis).second)
535 return emitError() << "grid axis duplicated";
536 }
537 return success();
538 };
539
540 for (auto subAxes : getSplitAxes().getAxes()) {
541 ArrayRef<GridAxis> subAxesArray = subAxes.asArrayRef();
542 if (failed(Result: checkGridAxis(subAxesArray)))
543 return failure();
544 }
545
546 if (!getStaticHaloSizes().empty() && !getStaticShardedDimsOffsets().empty()) {
547 return emitOpError(message: "halo sizes and shard offsets are mutually exclusive");
548 }
549
550 if (!getStaticHaloSizes().empty()) {
551 auto numSplitAxes = getSplitAxes().getAxes().size();
552 for (auto splitAxis : getSplitAxes().getAxes()) {
553 if (splitAxis.empty()) {
554 --numSplitAxes;
555 }
556 }
557 if (getStaticHaloSizes().size() != numSplitAxes * 2) {
558 return emitError() << "halo sizes must be specified for all split axes.";
559 }
560 }
561
562 return success();
563}
564
565void ShardingOp::getAsmResultNames(
566 function_ref<void(Value, StringRef)> setNameFn) {
567 setNameFn(getResult(), "sharding");
568}
569
570LogicalResult ShardingOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
571 auto grid = ::getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
572 if (failed(Result: grid)) {
573 return failure();
574 }
575 if (mlir::ShapedType::isDynamicShape(dSizes: grid->getShape()) &&
576 !getStaticShardedDimsOffsets().empty()) {
577 return emitError() << "sharded dims offsets are not allowed for "
578 "device grids with dynamic shape.";
579 }
580
581 auto shardedDimsOffsets = getStaticShardedDimsOffsets();
582 if (!shardedDimsOffsets.empty()) {
583 auto gridShape = grid.value().getShape();
584 assert(ShapedType::isStaticShape(gridShape));
585 uint64_t pos = 0;
586 for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(First: getSplitAxes())) {
587 if (!innerSplitAxes.empty()) {
588 int64_t numShards = 0, off = 0;
589 for (auto i : innerSplitAxes.asArrayRef()) {
590 numShards += gridShape[i];
591 }
592 for (int64_t i = 0; i <= numShards; ++i) {
593 if (shardedDimsOffsets.size() <= pos + i) {
594 return emitError() << "sharded dims offsets has wrong size.";
595 }
596 if (ShapedType::isStatic(dValue: shardedDimsOffsets[pos + i])) {
597 if (shardedDimsOffsets[pos + i] < off) {
598 return emitError()
599 << "sharded dims offsets must be non-decreasing.";
600 }
601 off = shardedDimsOffsets[pos + i];
602 }
603 }
604 pos += numShards + 1;
605 }
606 }
607 }
608 return success();
609}
610
611namespace {
612// Sharding annotations "halo sizes" and "sharded dims offsets"
613// are a mix of attributes and dynamic values. This canonicalization moves
614// constant values to the respective attribute lists, minimizing the number
615// of values.
616// It also removes sharded_dims_sizes and halos if they are effectively "empty".
617class NormalizeSharding final : public OpRewritePattern<ShardingOp> {
618public:
619 using OpRewritePattern<ShardingOp>::OpRewritePattern;
620
621 LogicalResult matchAndRewrite(ShardingOp op,
622 PatternRewriter &b) const override {
623 auto mixedHalos =
624 getMixedValues(staticValues: op.getStaticHaloSizes(), dynamicValues: op.getDynamicHaloSizes(), b);
625 auto mixedOffs = getMixedValues(staticValues: op.getStaticShardedDimsOffsets(),
626 dynamicValues: op.getDynamicShardedDimsOffsets(), b);
627
628 // No constant operands were folded, just return;
629 bool modified = succeeded(Result: foldDynamicIndexList(ofrs&: mixedHalos, onlyNonNegative: true)) ||
630 succeeded(Result: foldDynamicIndexList(ofrs&: mixedOffs, onlyNonNegative: true));
631
632 auto decomposedHalos = decomposeMixedValues(mixedValues: mixedHalos);
633 auto staticHalos = decomposedHalos.first;
634 auto dynamicHalos = decomposedHalos.second;
635 auto decomposedOffs = decomposeMixedValues(mixedValues: mixedOffs);
636 auto staticOffs = decomposedOffs.first;
637 auto dynamicOffs = decomposedOffs.second;
638
639 if (dynamicHalos.empty() && !staticHalos.empty()) {
640 if (staticHalos[0] == 0 && llvm::all_equal(Range&: staticHalos)) {
641 staticHalos.clear();
642 modified = true;
643 }
644 }
645
646 // Remove sharded dims offsets if they are effectively the default values,
647 // e.g. if they define equi-distance between all neighboring shards.
648 // Requires static-only offsets. Compares the first distance as the
649 // difference between the first two offsets. Only if all consecutive
650 // distances are the same, the offsets are removed.
651 if (dynamicOffs.empty() && !staticOffs.empty()) {
652 assert(staticOffs.size() >= 2);
653 auto diff = staticOffs[1] - staticOffs[0];
654 bool allSame = staticOffs.size() > 2;
655 for (auto i = 2u; i < staticOffs.size(); ++i) {
656 if (staticOffs[i] - staticOffs[i - 1] != diff) {
657 allSame = false;
658 break;
659 }
660 }
661 if (allSame) {
662 staticOffs.clear();
663 modified = true;
664 }
665 }
666
667 if (!modified) {
668 return failure();
669 }
670
671 b.modifyOpInPlace(root: op, callable: [&]() {
672 op.setStaticHaloSizes(staticHalos);
673 op.getDynamicHaloSizesMutable().assign(values: dynamicHalos);
674 op.setStaticShardedDimsOffsets(staticOffs);
675 op.getDynamicShardedDimsOffsetsMutable().assign(values: dynamicOffs);
676 });
677 return success();
678 }
679};
680} // namespace
681
682void ShardingOp::getCanonicalizationPatterns(mlir::RewritePatternSet &results,
683 mlir::MLIRContext *context) {
684 results.add<NormalizeSharding>(arg&: context);
685}
686
687//===----------------------------------------------------------------------===//
688// Sharding
689//===----------------------------------------------------------------------===//
690
691bool Sharding::equalSplitAxes(const Sharding &rhs) const {
692 if (getGrid() != rhs.getGrid()) {
693 return false;
694 }
695
696 auto minSize = std::min(a: getSplitAxes().size(), b: rhs.getSplitAxes().size());
697 if (!llvm::equal(LRange: llvm::make_range(x: getSplitAxes().begin(),
698 y: getSplitAxes().begin() + minSize),
699 RRange: llvm::make_range(x: rhs.getSplitAxes().begin(),
700 y: rhs.getSplitAxes().begin() + minSize))) {
701 return false;
702 }
703
704 return llvm::all_of(Range: llvm::drop_begin(RangeOrContainer: getSplitAxes(), N: minSize),
705 P: std::mem_fn(pm: &GridAxesAttr::empty)) &&
706 llvm::all_of(Range: llvm::drop_begin(RangeOrContainer: rhs.getSplitAxes(), N: minSize),
707 P: std::mem_fn(pm: &GridAxesAttr::empty));
708}
709
710bool Sharding::equalHaloAndShardSizes(const Sharding &rhs) const {
711 return equalShardSizes(rhs) && equalHaloSizes(rhs);
712}
713
714bool Sharding::equalShardSizes(const Sharding &rhs) const {
715 if (rhs.getStaticShardedDimsOffsets().size() !=
716 getStaticShardedDimsOffsets().size() ||
717 !llvm::equal(LRange: getStaticShardedDimsOffsets(),
718 RRange: rhs.getStaticShardedDimsOffsets())) {
719 return false;
720 }
721 if (rhs.getDynamicShardedDimsOffsets().size() !=
722 getDynamicShardedDimsOffsets().size() ||
723 !llvm::equal(LRange: getDynamicShardedDimsOffsets(),
724 RRange: rhs.getDynamicShardedDimsOffsets())) {
725 return false;
726 }
727 return true;
728}
729
730bool Sharding::equalHaloSizes(const Sharding &rhs) const {
731 if (rhs.getStaticHaloSizes().size() != getStaticHaloSizes().size() ||
732 !llvm::equal(LRange: getStaticHaloSizes(), RRange: rhs.getStaticHaloSizes())) {
733 return false;
734 }
735 if (rhs.getDynamicHaloSizes().size() != getDynamicHaloSizes().size() ||
736 !llvm::equal(LRange: getDynamicHaloSizes(), RRange: rhs.getDynamicHaloSizes())) {
737 return false;
738 }
739 return true;
740}
741
742bool Sharding::operator==(Value rhs) const {
743 return equalSplitAxes(rhs) && equalHaloAndShardSizes(rhs);
744}
745
746bool Sharding::operator!=(Value rhs) const { return !(*this == rhs); }
747
748bool Sharding::operator==(const Sharding &rhs) const {
749 return equalSplitAxes(rhs) && equalHaloAndShardSizes(rhs);
750}
751
752bool Sharding::operator!=(const Sharding &rhs) const { return !(*this == rhs); }
753
754llvm::raw_ostream &mlir::shard::operator<<(llvm::raw_ostream &os,
755 const Sharding &sharding) {
756 os << "Sharding<grid=" << sharding.getGrid() << ", split_axes=[";
757 llvm::interleaveComma(c: sharding.getSplitAxes(), os, each_fn: [&](GridAxesAttr axes) {
758 os << "[";
759 llvm::interleaveComma(c: axes.asArrayRef(), os);
760 os << "]";
761 });
762 os << "]";
763 if (!sharding.getStaticHaloSizes().empty()) {
764 os << ", halo_sizes=[";
765 llvm::interleaveComma(c: sharding.getStaticHaloSizes(), os);
766 os << "]";
767 }
768 if (!sharding.getStaticShardedDimsOffsets().empty()) {
769 os << ", sharded_dims_offsets=[";
770 llvm::interleaveComma(c: sharding.getStaticShardedDimsOffsets(), os);
771 os << "]";
772 }
773 os << ">";
774 return os;
775}
776
777Sharding::Sharding(::mlir::FlatSymbolRefAttr grid) : grid(grid) {}
778
779Sharding::Sharding(Value rhs) {
780 auto shardingOp = rhs.getDefiningOp<ShardingOp>();
781 assert(shardingOp && "expected sharding op");
782 auto splitAxes = shardingOp.getSplitAxes().getAxes();
783 // If splitAxes are empty, use "empty" constructor.
784 if (splitAxes.empty()) {
785 *this = Sharding(shardingOp.getGridAttr());
786 return;
787 }
788 *this =
789 get(grid_: shardingOp.getGridAttr(), split_axes_: splitAxes, static_halo_sizes_: shardingOp.getStaticHaloSizes(),
790 static_sharded_dims_offsets_: shardingOp.getStaticShardedDimsOffsets(),
791 dynamic_halo_sizes_: SmallVector<Value>(shardingOp.getDynamicHaloSizes()),
792 dynamic_sharded_dims_offsets_: SmallVector<Value>(shardingOp.getDynamicShardedDimsOffsets()));
793}
794
795Sharding Sharding::get(::mlir::FlatSymbolRefAttr grid,
796 ArrayRef<GridAxesAttr> splitAxes,
797 ArrayRef<int64_t> staticHaloSizes,
798 ArrayRef<int64_t> staticShardedDimsOffsets,
799 ArrayRef<Value> dynamicHaloSizes,
800 ArrayRef<Value> dynamicShardedDimsOffsets) {
801 Sharding res(grid);
802 if (splitAxes.empty()) {
803 return res;
804 }
805
806 res.split_axes.resize(N: splitAxes.size());
807 for (auto [i, axis] : llvm::enumerate(First&: splitAxes)) {
808 res.split_axes[i] = GridAxesAttr::get(context: grid.getContext(), content: axis.asArrayRef());
809 }
810
811 auto clone = [](const auto src, auto &dst) {
812 dst.resize(src.size());
813 llvm::copy(src, dst.begin());
814 };
815
816 clone(staticHaloSizes, res.static_halo_sizes);
817 clone(staticShardedDimsOffsets, res.static_sharded_dims_offsets);
818 clone(dynamicHaloSizes, res.dynamic_halo_sizes);
819 clone(dynamicShardedDimsOffsets, res.dynamic_sharded_dims_offsets);
820
821 return res;
822}
823
824//===----------------------------------------------------------------------===//
825// shard.shard_shape
826//===----------------------------------------------------------------------===//
827
828void ShardShapeOp::getAsmResultNames(
829 function_ref<void(Value, StringRef)> setNameFn) {
830 setNameFn(getResult()[0], "shard_shape");
831}
832
833void ShardShapeOp::build(::mlir::OpBuilder &odsBuilder,
834 ::mlir::OperationState &odsState,
835 ::llvm::ArrayRef<int64_t> dims,
836 ArrayRef<Value> dimsDyn, ::mlir::Value sharding,
837 ::mlir::ValueRange device) {
838 SmallVector<mlir::Type> resType(dims.size(), odsBuilder.getIndexType());
839 build(odsBuilder, odsState, result: resType, dims, dims_dynamic: dimsDyn, sharding,
840 device: SmallVector<int64_t>(device.size(), ShapedType::kDynamic), device_dynamic: device);
841}
842
843//===----------------------------------------------------------------------===//
844// shard.shard op
845//===----------------------------------------------------------------------===//
846
847void ShardOp::getAsmResultNames(
848 function_ref<void(Value, StringRef)> setNameFn) {
849 setNameFn(getResult(), "sharding_annotated");
850}
851
852namespace {
853// Determine if the given ShardOp is a duplicate of another ShardOp
854// on the same value. This can happen if constant values are sharded.
855class FoldDuplicateShardOp final : public OpRewritePattern<ShardOp> {
856public:
857 using OpRewritePattern<ShardOp>::OpRewritePattern;
858
859 LogicalResult matchAndRewrite(ShardOp op, PatternRewriter &b) const override {
860 // Get the use-list of the value being sharded and check if it has more than
861 // one use.
862 Value value = op.getSrc();
863 if (value.hasOneUse() || value.getDefiningOp<ShardOp>()) {
864 return failure();
865 }
866
867 // Iterate through the uses of the value to find a duplicate ShardOp.
868 for (auto &use : value.getUses()) {
869 if (use.getOwner() != op.getOperation()) {
870 auto otherOp = dyn_cast<ShardOp>(Val: use.getOwner());
871 // Only fold duplicates in the same block; isBeforeInBlock requires it.
872 if (!otherOp || otherOp->getBlock() != op->getBlock() ||
873 !otherOp->isBeforeInBlock(other: op)) {
874 return failure();
875 }
876 // Create a Sharding object for the current and the other ShardOp
877 // If the two are equal replace current op with the other op.
878 Sharding currentSharding(op.getSharding());
879 Sharding otherSharding(otherOp.getSharding());
880 if (currentSharding == otherSharding) {
881 b.replaceAllUsesWith(from: op.getResult(), to: otherOp.getResult());
882 b.eraseOp(op: op.getOperation());
883 } else {
884 // use the other sharding as input for op
885 b.modifyOpInPlace(
886 root: op, callable: [&]() { op.getSrcMutable().assign(value: otherOp.getResult()); });
887 }
888 return success();
889 }
890 }
891
892 return failure();
893 }
894};
895} // namespace
896
897void ShardOp::getCanonicalizationPatterns(mlir::RewritePatternSet &results,
898 mlir::MLIRContext *context) {
899 results.add<FoldDuplicateShardOp>(arg&: context);
900}
901
902//===----------------------------------------------------------------------===//
903// shard.process_multi_index op
904//===----------------------------------------------------------------------===//
905
906LogicalResult
907ProcessMultiIndexOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
908 auto grid = ::getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
909 if (failed(Result: grid)) {
910 return failure();
911 }
912 if (failed(Result: verifyGridAxes(loc: getLoc(), axes: getAxes(), grid: grid.value()))) {
913 return failure();
914 }
915
916 size_t expectedResultsCount =
917 getAxes().empty() ? grid->getRank() : getAxes().size();
918 if (getResult().size() != expectedResultsCount) {
919 return emitError() << "Unexpected number of results " << getResult().size()
920 << ". Expected " << expectedResultsCount << ".";
921 }
922
923 return success();
924}
925
926void ProcessMultiIndexOp::build(OpBuilder &odsBuilder, OperationState &odsState,
927 GridOp grid) {
928 build(odsBuilder, odsState,
929 result: SmallVector<Type>(grid.getRank(), odsBuilder.getIndexType()),
930 grid: grid.getSymName(), axes: ArrayRef<GridAxis>());
931}
932
933void ProcessMultiIndexOp::build(OpBuilder &odsBuilder, OperationState &odsState,
934 StringRef grid, ArrayRef<GridAxis> axes) {
935 build(odsBuilder, odsState,
936 result: SmallVector<Type>(axes.size(), odsBuilder.getIndexType()), grid,
937 axes: GridAxesAttr::get(context: odsBuilder.getContext(), content: axes));
938}
939
940void ProcessMultiIndexOp::getAsmResultNames(
941 function_ref<void(Value, StringRef)> setNameFn) {
942 setNameFn(getResults()[0], "proc_linear_idx");
943}
944
945//===----------------------------------------------------------------------===//
946// shard.process_linear_index op
947//===----------------------------------------------------------------------===//
948
949LogicalResult
950ProcessLinearIndexOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
951 auto grid = ::getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
952 if (failed(Result: grid)) {
953 return failure();
954 }
955 return success();
956}
957
958void ProcessLinearIndexOp::build(OpBuilder &odsBuilder,
959 OperationState &odsState, GridOp grid) {
960 build(odsBuilder, odsState, grid: grid.getSymName());
961}
962
963void ProcessLinearIndexOp::getAsmResultNames(
964 function_ref<void(Value, StringRef)> setNameFn) {
965 setNameFn(getResult(), "proc_linear_idx");
966}
967
968//===----------------------------------------------------------------------===//
969// shard.neighbors_linear_indices op
970//===----------------------------------------------------------------------===//
971
972LogicalResult
973NeighborsLinearIndicesOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
974 auto grid = ::getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
975 if (failed(Result: grid)) {
976 return failure();
977 }
978 return success();
979}
980
981void NeighborsLinearIndicesOp::getAsmResultNames(
982 function_ref<void(Value, StringRef)> setNameFn) {
983 setNameFn(getNeighborDown(), "down_linear_idx");
984 setNameFn(getNeighborUp(), "up_linear_idx");
985}
986
987//===----------------------------------------------------------------------===//
988// collective communication ops
989//===----------------------------------------------------------------------===//
990
991namespace {
992
993template <typename Op>
994struct EmptyGridAxesCanonicalizationPattern : OpRewritePattern<Op> {
995 using OpRewritePattern<Op>::OpRewritePattern;
996 LogicalResult matchAndRewrite(Op op,
997 PatternRewriter &rewriter) const override {
998 auto gridAxes = op.getGridAxes();
999 if (!gridAxes.empty()) {
1000 return failure();
1001 }
1002 if (op.getInput().getType() != op.getResult().getType()) {
1003 return failure();
1004 }
1005
1006 rewriter.replaceAllUsesWith(op.getResult(), op.getInput());
1007 rewriter.eraseOp(op: op.getOperation());
1008 return success();
1009 }
1010};
1011
1012} // namespace
1013
1014static LogicalResult verifyInGroupDevice(Location loc, StringRef deviceName,
1015 ArrayRef<int64_t> device,
1016 Operation::operand_range deviceDynamic,
1017 ArrayRef<GridAxis> gridAxes,
1018 ArrayRef<int64_t> gridShape) {
1019 if (device.size() != gridAxes.size()) {
1020 return emitError(loc) << "In-group device \"" << deviceName
1021 << "\" has unexpected multi-index size "
1022 << device.size() << ". Expected " << gridAxes.size()
1023 << ".";
1024 }
1025
1026 for (size_t i = 0; i < device.size(); ++i) {
1027 if (ShapedType::isStatic(dValue: device[i]) &&
1028 ShapedType::isStatic(dValue: gridShape[gridAxes[i]]) &&
1029 gridShape[gridAxes[i]] <= device[i]) {
1030 return emitError(loc)
1031 << "Out of bounds coordinate " << i << " for in-group device \""
1032 << deviceName << "\"."
1033 << " Got " << device[i] << ", but expected value in the range [0, "
1034 << (gridShape[gridAxes[i]] - 1) << "].";
1035 }
1036 }
1037 return success();
1038}
1039
1040static LogicalResult verifyDimensionCompatibility(Location loc,
1041 int64_t expectedDimSize,
1042 int64_t resultDimSize,
1043 int64_t resultAxis) {
1044 if (ShapedType::isStatic(dValue: resultDimSize) && expectedDimSize != resultDimSize) {
1045 return emitError(loc) << "Dimension size mismatch for result axis "
1046 << resultAxis << ". Expected "
1047 << (ShapedType::isDynamic(dValue: expectedDimSize)
1048 ? Twine("dynamic")
1049 : Twine(expectedDimSize))
1050 << ", but got " << resultDimSize << ".";
1051 }
1052
1053 return success();
1054}
1055
1056static LogicalResult verifyGatherOperandAndResultShape(
1057 Value operand, Value result, int64_t gatherAxis,
1058 ArrayRef<GridAxis> gridAxes, ArrayRef<int64_t> gridShape) {
1059 auto resultRank = cast<ShapedType>(Val: result.getType()).getRank();
1060 if (gatherAxis < 0 || gatherAxis >= resultRank) {
1061 return emitError(loc: result.getLoc())
1062 << "Gather axis " << gatherAxis << " is out of bounds [0, "
1063 << resultRank << ").";
1064 }
1065
1066 ShapedType operandType = cast<ShapedType>(Val: operand.getType());
1067 ShapedType resultType = cast<ShapedType>(Val: result.getType());
1068 auto deviceGroupSize =
1069 DimensionSize(collectiveProcessGroupSize(gridAxes, gridShape));
1070 for (int64_t axis = 0; axis < operandType.getRank(); ++axis) {
1071 auto operandDimSize = DimensionSize(operandType.getDimSize(idx: axis));
1072 auto resultDimSize = DimensionSize(resultType.getDimSize(idx: axis));
1073 auto expectedResultDimSize =
1074 axis == gatherAxis ? deviceGroupSize * operandDimSize : operandDimSize;
1075 if (failed(Result: verifyDimensionCompatibility(
1076 loc: result.getLoc(), expectedDimSize: expectedResultDimSize, resultDimSize, resultAxis: axis))) {
1077 return failure();
1078 }
1079 }
1080 return success();
1081}
1082
1083static LogicalResult verifyAllToAllOperandAndResultShape(
1084 Value operand, Value result, int64_t splitAxis, int64_t concatAxis,
1085 ArrayRef<GridAxis> gridAxes, ArrayRef<int64_t> gridShape) {
1086 ShapedType operandType = cast<ShapedType>(Val: operand.getType());
1087 ShapedType resultType = cast<ShapedType>(Val: result.getType());
1088 auto operandRank = operandType.getRank();
1089 if (splitAxis < 0 || splitAxis >= operandRank) {
1090 return emitError(loc: result.getLoc())
1091 << "split_axis " << splitAxis << " is out of bounds [0, "
1092 << operandRank << ").";
1093 }
1094 if (concatAxis < 0 || concatAxis >= operandRank) {
1095 return emitError(loc: result.getLoc())
1096 << "concat_axis " << concatAxis << " is out of bounds [0, "
1097 << operandRank << ").";
1098 }
1099 for (int64_t axis = 0; axis < operandType.getRank(); ++axis) {
1100 if ((axis != splitAxis && axis != concatAxis) || splitAxis == concatAxis) {
1101 if (failed(Result: verifyDimensionCompatibility(
1102 loc: result.getLoc(), expectedDimSize: operandType.getDimSize(idx: axis),
1103 resultDimSize: resultType.getDimSize(idx: axis), resultAxis: axis))) {
1104 return failure();
1105 }
1106 }
1107 }
1108
1109 if (splitAxis == concatAxis) {
1110 return success();
1111 }
1112
1113 auto deviceGroupSize =
1114 DimensionSize(collectiveProcessGroupSize(gridAxes, gridShape));
1115 auto operandConcatDimSize = DimensionSize(operandType.getDimSize(idx: concatAxis));
1116 auto operandSplitDimSize = DimensionSize(operandType.getDimSize(idx: splitAxis));
1117 DimensionSize expectedResultConcatDimSize =
1118 operandConcatDimSize * deviceGroupSize;
1119 DimensionSize expectedResultSplitDimSize =
1120 operandSplitDimSize / deviceGroupSize;
1121 if (!expectedResultSplitDimSize.isDynamic() &&
1122 int64_t(operandSplitDimSize) % int64_t(deviceGroupSize) != 0) {
1123 expectedResultSplitDimSize = DimensionSize::dynamic();
1124 }
1125 if (failed(Result: verifyDimensionCompatibility(
1126 loc: result.getLoc(), expectedDimSize: expectedResultConcatDimSize.value(),
1127 resultDimSize: resultType.getDimSize(idx: concatAxis), resultAxis: concatAxis))) {
1128 return failure();
1129 }
1130 if (failed(Result: verifyDimensionCompatibility(
1131 loc: result.getLoc(), expectedDimSize: expectedResultSplitDimSize.value(),
1132 resultDimSize: resultType.getDimSize(idx: splitAxis), resultAxis: splitAxis))) {
1133 return failure();
1134 }
1135
1136 return success();
1137}
1138
1139static LogicalResult verifyScatterOrSliceOperandAndResultShape(
1140 Value operand, Value result, int64_t tensorAxis,
1141 StringRef tensorAxisAttrName, ArrayRef<GridAxis> gridAxes,
1142 ArrayRef<int64_t> gridShape) {
1143 ShapedType operandType = cast<ShapedType>(Val: operand.getType());
1144 ShapedType resultType = cast<ShapedType>(Val: result.getType());
1145 auto operandRank = operandType.getRank();
1146 if (tensorAxis < 0 || tensorAxis >= operandRank) {
1147 return emitError(loc: result.getLoc())
1148 << tensorAxisAttrName << " " << tensorAxis
1149 << " is out of bounds [0, " << operandRank << ").";
1150 }
1151 for (int64_t axis = 0; axis < operandType.getRank(); ++axis) {
1152 if (axis != tensorAxis) {
1153 if (failed(Result: verifyDimensionCompatibility(
1154 loc: result.getLoc(), expectedDimSize: operandType.getDimSize(idx: axis),
1155 resultDimSize: resultType.getDimSize(idx: axis), resultAxis: axis))) {
1156 return failure();
1157 }
1158 }
1159 }
1160
1161 auto deviceGroupSize =
1162 DimensionSize(collectiveProcessGroupSize(gridAxes, gridShape));
1163 auto operandScatterDimSize =
1164 DimensionSize(operandType.getDimSize(idx: tensorAxis));
1165 if (!operandScatterDimSize.isDynamic() && !deviceGroupSize.isDynamic() &&
1166 int64_t(operandScatterDimSize) % int64_t(deviceGroupSize) != 0) {
1167 return emitError(loc: result.getLoc())
1168 << "Operand dimension size " << int64_t(operandScatterDimSize)
1169 << " is not divisible by collective device group size "
1170 << int64_t(deviceGroupSize) << " for tensor axis " << tensorAxis
1171 << ".";
1172 }
1173 DimensionSize expectedResultTensorDimSize =
1174 operandScatterDimSize / deviceGroupSize;
1175 if (failed(Result: verifyDimensionCompatibility(
1176 loc: result.getLoc(), expectedDimSize: expectedResultTensorDimSize.value(),
1177 resultDimSize: resultType.getDimSize(idx: tensorAxis), resultAxis: tensorAxis))) {
1178 return failure();
1179 }
1180
1181 return success();
1182}
1183
1184static RankedTensorType sliceResultType(Type operandType, GridOp grid,
1185 ArrayRef<GridAxis> gridAxes,
1186 int64_t sliceAxis) {
1187 RankedTensorType operandRankedTensorType =
1188 cast<RankedTensorType>(Val&: operandType);
1189 DimensionSize operandSliceAxisSize =
1190 operandRankedTensorType.getShape()[sliceAxis];
1191 SmallVector<int64_t> resultShape =
1192 llvm::to_vector(Range: operandRankedTensorType.getShape());
1193
1194 resultShape[sliceAxis] =
1195 operandSliceAxisSize /
1196 DimensionSize(collectiveProcessGroupSize(gridAxes, grid));
1197 return operandRankedTensorType.clone(shape: resultShape);
1198}
1199
1200//===----------------------------------------------------------------------===//
1201// shard.all_gather op
1202//===----------------------------------------------------------------------===//
1203
1204LogicalResult
1205AllGatherOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1206 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1207 if (failed(Result: grid)) {
1208 return failure();
1209 }
1210 auto gatherAxis = getGatherAxis().getSExtValue();
1211 return verifyGatherOperandAndResultShape(operand: getOperand(), result: getResult(),
1212 gatherAxis, gridAxes: getGridAxes(),
1213 gridShape: grid.value().getShape());
1214}
1215
1216void AllGatherOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1217 MLIRContext *context) {
1218 patterns.add<EmptyGridAxesCanonicalizationPattern<AllGatherOp>>(arg&: context);
1219}
1220
1221void AllGatherOp::getAsmResultNames(
1222 function_ref<void(Value, StringRef)> setNameFn) {
1223 setNameFn(getResult(), "all_gather");
1224}
1225
1226//===----------------------------------------------------------------------===//
1227// shard.all_reduce op
1228//===----------------------------------------------------------------------===//
1229
1230LogicalResult
1231AllReduceOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1232 return getGridAndVerifyAxes(op: *this, symbolTable);
1233}
1234
1235void AllReduceOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1236 MLIRContext *context) {
1237 patterns.add<EmptyGridAxesCanonicalizationPattern<AllReduceOp>>(arg&: context);
1238}
1239
1240void AllReduceOp::build(OpBuilder &odsBuilder, OperationState &odsState,
1241 Value input, StringRef grid,
1242 ArrayRef<GridAxis> gridAxes, ReductionKind reduction) {
1243 build(odsBuilder, odsState, result: input.getType(), grid, grid_axes: gridAxes, input,
1244 reduction);
1245}
1246
1247void AllReduceOp::getAsmResultNames(
1248 function_ref<void(Value, StringRef)> setNameFn) {
1249 setNameFn(getResult(), "all_reduce");
1250}
1251
1252//===----------------------------------------------------------------------===//
1253// shard.all_slice op
1254//===----------------------------------------------------------------------===//
1255
1256LogicalResult AllSliceOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1257 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1258 if (failed(Result: grid)) {
1259 return failure();
1260 }
1261 return verifyScatterOrSliceOperandAndResultShape(
1262 operand: getOperand(), result: getResult(), tensorAxis: getSliceAxis().getSExtValue(), tensorAxisAttrName: "slice_axis",
1263 gridAxes: getGridAxes(), gridShape: grid.value().getShape());
1264}
1265
1266void AllSliceOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1267 MLIRContext *context) {
1268 patterns.add<EmptyGridAxesCanonicalizationPattern<AllSliceOp>>(arg&: context);
1269}
1270
1271void AllSliceOp::build(OpBuilder &odsBuilder, OperationState &odsState,
1272 Value input, GridOp grid, ArrayRef<GridAxis> gridAxes,
1273 int64_t sliceAxis) {
1274 Type resultType = sliceResultType(operandType: input.getType(), grid, gridAxes, sliceAxis);
1275 build(odsBuilder, odsState, result_type: resultType, input, grid: grid.getSymName(), gridAxes,
1276 sliceAxis);
1277}
1278
1279void AllSliceOp::build(OpBuilder &odsBuilder, OperationState &odsState,
1280 Type resultType, Value input, StringRef grid,
1281 ArrayRef<GridAxis> gridAxes, int64_t sliceAxis) {
1282 build(odsBuilder, odsState, result: resultType, grid, grid_axes: gridAxes, input,
1283 slice_axis: APInt(sizeof(sliceAxis) * CHAR_BIT, sliceAxis));
1284}
1285
1286void AllSliceOp::getAsmResultNames(
1287 function_ref<void(Value, StringRef)> setNameFn) {
1288 setNameFn(getResult(), "all_slice");
1289}
1290
1291//===----------------------------------------------------------------------===//
1292// shard.all_to_all op
1293//===----------------------------------------------------------------------===//
1294
1295LogicalResult AllToAllOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1296 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1297 if (failed(Result: grid)) {
1298 return failure();
1299 }
1300
1301 return verifyAllToAllOperandAndResultShape(
1302 operand: getOperand(), result: getResult(), splitAxis: getSplitAxis().getSExtValue(),
1303 concatAxis: getConcatAxis().getSExtValue(), gridAxes: getGridAxes(), gridShape: grid.value().getShape());
1304}
1305
1306void AllToAllOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1307 MLIRContext *context) {
1308 patterns.add<EmptyGridAxesCanonicalizationPattern<AllToAllOp>>(arg&: context);
1309}
1310
1311void AllToAllOp::getAsmResultNames(
1312 function_ref<void(Value, StringRef)> setNameFn) {
1313 setNameFn(getResult(), "all_to_all");
1314}
1315
1316//===----------------------------------------------------------------------===//
1317// shard.broadcast op
1318//===----------------------------------------------------------------------===//
1319
1320LogicalResult
1321BroadcastOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1322 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1323 if (failed(Result: grid)) {
1324 return failure();
1325 }
1326 if (failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getRootAttrName(), device: getRoot(),
1327 deviceDynamic: getRootDynamic(), gridAxes: getGridAxes(),
1328 gridShape: grid.value().getShape()))) {
1329 return failure();
1330 }
1331
1332 return success();
1333}
1334
1335void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1336 MLIRContext *context) {
1337 patterns.add<EmptyGridAxesCanonicalizationPattern<BroadcastOp>>(arg&: context);
1338}
1339
1340void BroadcastOp::getAsmResultNames(
1341 function_ref<void(Value, StringRef)> setNameFn) {
1342 setNameFn(getResult(), "broadcast");
1343}
1344
1345//===----------------------------------------------------------------------===//
1346// shard.gather op
1347//===----------------------------------------------------------------------===//
1348
1349LogicalResult GatherOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1350 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1351 if (failed(Result: grid)) {
1352 return failure();
1353 }
1354 if (failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getRootAttrName(), device: getRoot(),
1355 deviceDynamic: getRootDynamic(), gridAxes: getGridAxes(),
1356 gridShape: grid.value().getShape()))) {
1357 return failure();
1358 }
1359
1360 auto gatherAxis = getGatherAxis().getSExtValue();
1361 return verifyGatherOperandAndResultShape(operand: getInput(), result: getResult(), gatherAxis,
1362 gridAxes: getGridAxes(),
1363 gridShape: grid.value().getShape());
1364}
1365
1366void GatherOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1367 MLIRContext *context) {
1368 patterns.add<EmptyGridAxesCanonicalizationPattern<GatherOp>>(arg&: context);
1369}
1370
1371void GatherOp::getAsmResultNames(
1372 function_ref<void(Value, StringRef)> setNameFn) {
1373 setNameFn(getResult(), "gather");
1374}
1375
1376//===----------------------------------------------------------------------===//
1377// shard.recv op
1378//===----------------------------------------------------------------------===//
1379
1380LogicalResult RecvOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1381 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1382 if (failed(Result: grid)) {
1383 return failure();
1384 }
1385 if (getSource() &&
1386 failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getSourceAttrName(),
1387 device: getSource().value(), deviceDynamic: getSourceDynamic(),
1388 gridAxes: getGridAxes(), gridShape: grid.value().getShape()))) {
1389 return failure();
1390 }
1391 return success();
1392}
1393
1394void RecvOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1395 MLIRContext *context) {
1396 patterns.add<EmptyGridAxesCanonicalizationPattern<RecvOp>>(arg&: context);
1397}
1398
1399void RecvOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
1400 setNameFn(getResult(), "recv");
1401}
1402
1403//===----------------------------------------------------------------------===//
1404// shard.reduce op
1405//===----------------------------------------------------------------------===//
1406
1407LogicalResult ReduceOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1408 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1409 if (failed(Result: grid)) {
1410 return failure();
1411 }
1412 if (failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getRootAttrName(), device: getRoot(),
1413 deviceDynamic: getRootDynamic(), gridAxes: getGridAxes(),
1414 gridShape: grid.value().getShape()))) {
1415 return failure();
1416 }
1417
1418 return success();
1419}
1420
1421void ReduceOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1422 MLIRContext *context) {
1423 patterns.add<EmptyGridAxesCanonicalizationPattern<ReduceOp>>(arg&: context);
1424}
1425
1426void ReduceOp::getAsmResultNames(
1427 function_ref<void(Value, StringRef)> setNameFn) {
1428 setNameFn(getResult(), "reduce");
1429}
1430
1431//===----------------------------------------------------------------------===//
1432// shard.reduce_scatter op
1433//===----------------------------------------------------------------------===//
1434
1435LogicalResult
1436ReduceScatterOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1437 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1438 if (failed(Result: grid)) {
1439 return failure();
1440 }
1441
1442 return verifyScatterOrSliceOperandAndResultShape(
1443 operand: getOperand(), result: getResult(), tensorAxis: getScatterDim().getSExtValue(), tensorAxisAttrName: "scatter_dim",
1444 gridAxes: getGridAxes(), gridShape: grid.value().getShape());
1445}
1446
1447void ReduceScatterOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1448 MLIRContext *context) {
1449 patterns.add<EmptyGridAxesCanonicalizationPattern<ReduceScatterOp>>(arg&: context);
1450}
1451
1452void ReduceScatterOp::getAsmResultNames(
1453 function_ref<void(Value, StringRef)> setNameFn) {
1454 setNameFn(getResult(), "reduce_scatter");
1455}
1456
1457//===----------------------------------------------------------------------===//
1458// shard.scatter op
1459//===----------------------------------------------------------------------===//
1460
1461LogicalResult ScatterOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1462 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1463 if (failed(Result: grid)) {
1464 return failure();
1465 }
1466 if (failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getRootAttrName(), device: getRoot(),
1467 deviceDynamic: getRootDynamic(), gridAxes: getGridAxes(),
1468 gridShape: grid.value().getShape()))) {
1469 return failure();
1470 }
1471
1472 auto scatterDim = getScatterDim().getSExtValue();
1473 return verifyScatterOrSliceOperandAndResultShape(
1474 operand: getInput(), result: getResult(), tensorAxis: scatterDim, tensorAxisAttrName: "scatter_dim", gridAxes: getGridAxes(),
1475 gridShape: grid.value().getShape());
1476}
1477
1478void ScatterOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1479 MLIRContext *context) {
1480 patterns.add<EmptyGridAxesCanonicalizationPattern<ScatterOp>>(arg&: context);
1481}
1482
1483void ScatterOp::getAsmResultNames(
1484 function_ref<void(Value, StringRef)> setNameFn) {
1485 setNameFn(getResult(), "scatter");
1486}
1487
1488//===----------------------------------------------------------------------===//
1489// shard.send op
1490//===----------------------------------------------------------------------===//
1491
1492LogicalResult SendOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1493 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1494 if (failed(Result: grid)) {
1495 return failure();
1496 }
1497 if (failed(Result: verifyInGroupDevice(loc: getLoc(), deviceName: getDestinationAttrName(),
1498 device: getDestination(), deviceDynamic: getDestinationDynamic(),
1499 gridAxes: getGridAxes(), gridShape: grid.value().getShape()))) {
1500 return failure();
1501 }
1502 return success();
1503}
1504
1505void SendOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1506 MLIRContext *context) {
1507 patterns.add<EmptyGridAxesCanonicalizationPattern<SendOp>>(arg&: context);
1508}
1509
1510void SendOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
1511 setNameFn(getResult(), "send");
1512}
1513
1514//===----------------------------------------------------------------------===//
1515// shard.shift op
1516//===----------------------------------------------------------------------===//
1517
1518LogicalResult ShiftOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1519 auto grid = getGridAndVerifyAxes(op: *this, symbolTable);
1520 if (failed(Result: grid)) {
1521 return failure();
1522 }
1523
1524 auto gridAxes = getGridAxes();
1525 auto shiftAxis = getShiftAxis().getZExtValue();
1526 if (!llvm::is_contained(Range&: gridAxes, Element: shiftAxis)) {
1527 return emitError() << "Invalid shift axis " << shiftAxis
1528 << ". It must be one of the grouping grid axes.";
1529 }
1530
1531 return success();
1532}
1533
1534void ShiftOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1535 MLIRContext *context) {
1536 // TODO: remove op when offset is 0 or if it is a rotate with and
1537 // offset % shift_axis_grid_dim_size == 0.
1538}
1539
1540void ShiftOp::getAsmResultNames(
1541 function_ref<void(Value, StringRef)> setNameFn) {
1542 setNameFn(getResult(), "shift");
1543}
1544
1545//===----------------------------------------------------------------------===//
1546// shard.update_halo op
1547//===----------------------------------------------------------------------===//
1548
1549LogicalResult
1550UpdateHaloOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1551 auto grid = getGridAndVerify(op: getOperation(), gridSymbol: getGridAttr(), symbolTable);
1552 if (failed(Result: grid)) {
1553 return failure();
1554 }
1555
1556 return success();
1557}
1558
1559//===----------------------------------------------------------------------===//
1560// TableGen'd op method definitions
1561//===----------------------------------------------------------------------===//
1562
1563#define GET_OP_CLASSES
1564#include "mlir/Dialect/Shard/IR/ShardOps.cpp.inc"
1565
1566#define GET_ATTRDEF_CLASSES
1567#include "mlir/Dialect/Shard/IR/ShardAttributes.cpp.inc"
1568
1569#define GET_TYPEDEF_CLASSES
1570#include "mlir/Dialect/Shard/IR/ShardTypes.cpp.inc"
1571
1572#include "mlir/Dialect/Shard/IR/ShardEnums.cpp.inc"
1573