| 1 | //===- Transforms.cpp ---------------------------------------------- C++ --===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Shard/Transforms/Transforms.h" |
| 10 | #include "TransformsDetail.h" |
| 11 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 12 | #include "mlir/Dialect/Affine/Utils.h" |
| 13 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 14 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 15 | #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h" |
| 16 | #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" |
| 17 | #include "mlir/Dialect/Shard/IR/ShardDialect.h" |
| 18 | #include "mlir/Dialect/Shard/IR/ShardOps.h" |
| 19 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 20 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 21 | #include "mlir/IR/BuiltinTypes.h" |
| 22 | #include "mlir/IR/DialectRegistry.h" |
| 23 | #include "mlir/IR/OpDefinition.h" |
| 24 | #include "mlir/IR/PatternMatch.h" |
| 25 | #include "mlir/IR/Value.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/SmallVector.h" |
| 28 | #include <iterator> |
| 29 | #include <numeric> |
| 30 | |
| 31 | namespace mlir::shard { |
| 32 | |
| 33 | namespace { |
| 34 | |
| 35 | /// Lower `shard.process_multi_index` into expression using |
| 36 | /// `shard.process_linear_index` and `shard.grid_shape`. |
| 37 | struct ProcessMultiIndexOpLowering |
| 38 | : OpRewritePatternWithSymbolTableCollection<ProcessMultiIndexOp> { |
| 39 | using OpRewritePatternWithSymbolTableCollection:: |
| 40 | OpRewritePatternWithSymbolTableCollection; |
| 41 | |
| 42 | LogicalResult matchAndRewrite(ProcessMultiIndexOp op, |
| 43 | PatternRewriter &rewriter) const override { |
| 44 | GridOp grid = getGrid(op, symbolTableCollection); |
| 45 | if (!grid) { |
| 46 | return failure(); |
| 47 | } |
| 48 | |
| 49 | ImplicitLocOpBuilder builder(op->getLoc(), rewriter); |
| 50 | builder.setInsertionPointAfter(op.getOperation()); |
| 51 | Value linearIndex = ProcessLinearIndexOp::create(builder, grid); |
| 52 | ValueRange gridShape = GridShapeOp::create(builder, grid).getResults(); |
| 53 | SmallVector<Value> completeMultiIndex = |
| 54 | affine::AffineDelinearizeIndexOp::create(builder, linear_index: linearIndex, |
| 55 | basis: gridShape) |
| 56 | .getMultiIndex(); |
| 57 | SmallVector<Value> multiIndex; |
| 58 | ArrayRef<GridAxis> opGridAxes = op.getAxes(); |
| 59 | SmallVector<GridAxis> opAxesIota; |
| 60 | if (opGridAxes.empty()) { |
| 61 | opAxesIota.resize(N: grid.getRank()); |
| 62 | std::iota(first: opAxesIota.begin(), last: opAxesIota.end(), value: 0); |
| 63 | opGridAxes = opAxesIota; |
| 64 | } |
| 65 | llvm::transform(Range&: opGridAxes, d_first: std::back_inserter(x&: multiIndex), |
| 66 | F: [&completeMultiIndex](GridAxis gridAxis) { |
| 67 | return completeMultiIndex[gridAxis]; |
| 68 | }); |
| 69 | rewriter.replaceOp(op, newValues: multiIndex); |
| 70 | return success(); |
| 71 | } |
| 72 | }; |
| 73 | |
| 74 | struct AllSliceOpLowering |
| 75 | : OpRewritePatternWithSymbolTableCollection<AllSliceOp> { |
| 76 | using OpRewritePatternWithSymbolTableCollection:: |
| 77 | OpRewritePatternWithSymbolTableCollection; |
| 78 | |
| 79 | LogicalResult matchAndRewrite(AllSliceOp op, |
| 80 | PatternRewriter &rewriter) const override { |
| 81 | // 1. Compute the process linear index inside the process group from its |
| 82 | // multi-index. |
| 83 | // |
| 84 | // 2. Extract a slice from the input tensor. |
| 85 | // All axes except the slicing axis are not interesting and take the full |
| 86 | // axis. |
| 87 | // The slice axis is split into equisized parts with count |
| 88 | // the number of processes in the collective process group induced by |
| 89 | // the grid axes. |
| 90 | // The part for each process is determined by the corresponding |
| 91 | // linear-index in the process group. |
| 92 | // |
| 93 | // There are no collectives that require communication. |
| 94 | // Each process operates on its local tensor. |
| 95 | |
| 96 | GridOp grid = getGrid(op, symbolTableCollection); |
| 97 | if (!grid) { |
| 98 | return failure(); |
| 99 | } |
| 100 | |
| 101 | ImplicitLocOpBuilder builder(op->getLoc(), rewriter); |
| 102 | builder.setInsertionPointAfter(op.getOperation()); |
| 103 | |
| 104 | Value zero = arith::ConstantOp::create(builder, value: builder.getIndexAttr(value: 0)); |
| 105 | |
| 106 | Operation::result_range processInGroupMultiIndex = |
| 107 | ProcessMultiIndexOp::create(builder, grid: grid.getSymName(), |
| 108 | axes: op.getGridAxes()) |
| 109 | .getResults(); |
| 110 | |
| 111 | Operation::result_range processGroupShape = |
| 112 | GridShapeOp::create(builder, grid: grid.getSymName(), axes: op.getGridAxes()) |
| 113 | .getResult(); |
| 114 | Value processGroupSize = |
| 115 | createCollectiveProcessGroupSize(grid, axes: op.getGridAxes(), builder); |
| 116 | |
| 117 | int64_t sliceAxis = op.getSliceAxis().getSExtValue(); |
| 118 | Value operandSliceAxisSize = |
| 119 | tensor::DimOp::create(builder, source: op.getOperand(), index: sliceAxis); |
| 120 | Value operandSliceAxisSizeModProcessGroupSize = |
| 121 | arith::RemUIOp::create(builder, lhs: operandSliceAxisSize, rhs: processGroupSize); |
| 122 | Value isTargetShapeExactlyDivisible = |
| 123 | arith::CmpIOp::create(builder, predicate: arith::CmpIPredicate::eq, |
| 124 | lhs: operandSliceAxisSizeModProcessGroupSize, rhs: zero); |
| 125 | cf::AssertOp::create(builder, arg: isTargetShapeExactlyDivisible, |
| 126 | msg: "Slicing a tensor with axis size that is " |
| 127 | "not exactly divisible by the " |
| 128 | "grid process group size is not supported." ); |
| 129 | Value resultSliceAxisSize = |
| 130 | arith::DivUIOp::create(builder, lhs: operandSliceAxisSize, rhs: processGroupSize); |
| 131 | OpFoldResult processInGroupLinearIndex = affine::linearizeIndex( |
| 132 | multiIndex: llvm::to_vector_of<OpFoldResult>(Range&: processInGroupMultiIndex), |
| 133 | basis: llvm::to_vector_of<OpFoldResult>(Range&: processGroupShape), builder); |
| 134 | |
| 135 | // insert tensor.extract_slice |
| 136 | RankedTensorType operandType = |
| 137 | cast<RankedTensorType>(Val: op.getOperand().getType()); |
| 138 | SmallVector<OpFoldResult> sizes; |
| 139 | for (int64_t i = 0; i < operandType.getRank(); ++i) { |
| 140 | if (i == sliceAxis) { |
| 141 | sizes.emplace_back(Args&: resultSliceAxisSize); |
| 142 | } else { |
| 143 | Value dimSize = tensor::DimOp::create(builder, source: op.getOperand(), index: i); |
| 144 | sizes.emplace_back(Args&: dimSize); |
| 145 | } |
| 146 | } |
| 147 | SmallVector<OpFoldResult> offsets( |
| 148 | operandType.getRank(), getAsIndexOpFoldResult(ctx: builder.getContext(), val: 0)); |
| 149 | offsets[sliceAxis] = |
| 150 | ArithBuilder(builder, builder.getLoc()) |
| 151 | .mul(lhs: getValueOrCreateConstantIndexOp(b&: builder, loc: builder.getLoc(), |
| 152 | ofr: processInGroupLinearIndex), |
| 153 | rhs: resultSliceAxisSize); |
| 154 | SmallVector<OpFoldResult> strides( |
| 155 | operandType.getRank(), getAsIndexOpFoldResult(ctx: builder.getContext(), val: 1)); |
| 156 | Value slice = tensor::ExtractSliceOp::create(builder, source: op.getOperand(), |
| 157 | offsets, sizes, strides); |
| 158 | Value newResult = |
| 159 | tensor::CastOp::create(builder, dest: op.getResult().getType(), source: slice); |
| 160 | rewriter.replaceOp(op, newValues: newResult); |
| 161 | return success(); |
| 162 | } |
| 163 | }; |
| 164 | |
| 165 | } // namespace |
| 166 | |
| 167 | void populateProcessMultiIndexOpLoweringPatterns( |
| 168 | RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) { |
| 169 | patterns.add<ProcessMultiIndexOpLowering>(arg&: symbolTableCollection, |
| 170 | args: patterns.getContext()); |
| 171 | } |
| 172 | |
| 173 | void registerProcessMultiIndexOpLoweringDialects(DialectRegistry ®istry) { |
| 174 | registry.insert<affine::AffineDialect, shard::ShardDialect>(); |
| 175 | } |
| 176 | |
| 177 | void populateAllSliceOpLoweringPatterns( |
| 178 | RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) { |
| 179 | patterns.add<AllSliceOpLowering>(arg&: symbolTableCollection, |
| 180 | args: patterns.getContext()); |
| 181 | } |
| 182 | |
| 183 | void registerAllSliceOpLoweringDialects(DialectRegistry ®istry) { |
| 184 | registry.insert<affine::AffineDialect, arith::ArithDialect, |
| 185 | cf::ControlFlowDialect, shard::ShardDialect, |
| 186 | tensor::TensorDialect>(); |
| 187 | } |
| 188 | |
| 189 | void populateAllOpLoweringPatterns( |
| 190 | RewritePatternSet &patterns, SymbolTableCollection &symbolTableCollection) { |
| 191 | populateProcessMultiIndexOpLoweringPatterns(patterns, symbolTableCollection); |
| 192 | populateAllSliceOpLoweringPatterns(patterns, symbolTableCollection); |
| 193 | } |
| 194 | |
| 195 | void registerAllOpLoweringDialects(DialectRegistry ®istry) { |
| 196 | registerProcessMultiIndexOpLoweringDialects(registry); |
| 197 | registerAllSliceOpLoweringDialects(registry); |
| 198 | } |
| 199 | |
| 200 | TypedValue<IndexType> |
| 201 | createCollectiveProcessGroupSize(GridOp grid, ArrayRef<GridAxis> axes, |
| 202 | ImplicitLocOpBuilder &builder) { |
| 203 | Operation::result_range gridShape = |
| 204 | GridShapeOp::create(builder, grid, axes).getResults(); |
| 205 | return cast<TypedValue<IndexType>>(Val: arith::createProduct( |
| 206 | builder, loc: builder.getLoc(), values: llvm::to_vector_of<Value>(Range&: gridShape), |
| 207 | resultType: builder.getIndexType())); |
| 208 | } |
| 209 | |
| 210 | TypedValue<IndexType> |
| 211 | createProcessLinearIndex(ImplicitLocOpBuilder &builder, StringRef grid, |
| 212 | ValueRange processInGroupMultiIndex, |
| 213 | ArrayRef<GridAxis> gridAxes) { |
| 214 | Operation::result_range processGroupShape = |
| 215 | GridShapeOp::create(builder, grid, axes: gridAxes).getResult(); |
| 216 | OpFoldResult processInGroupLinearIndex = affine::linearizeIndex( |
| 217 | multiIndex: llvm::to_vector_of<OpFoldResult>(Range&: processInGroupMultiIndex), |
| 218 | basis: llvm::to_vector_of<OpFoldResult>(Range&: processGroupShape), builder); |
| 219 | auto res = dyn_cast<Value>(Val&: processInGroupLinearIndex); |
| 220 | if (!res) |
| 221 | res = arith::ConstantIndexOp::create( |
| 222 | builder, |
| 223 | value: cast<IntegerAttr>(Val: cast<Attribute>(Val&: processInGroupLinearIndex)).getInt()); |
| 224 | return cast<TypedValue<IndexType>>(Val&: res); |
| 225 | } |
| 226 | |
| 227 | TypedValue<IndexType> createProcessLinearIndex(ImplicitLocOpBuilder &builder, |
| 228 | StringRef grid, |
| 229 | ArrayRef<GridAxis> gridAxes) { |
| 230 | return createProcessLinearIndex( |
| 231 | builder, grid, |
| 232 | processInGroupMultiIndex: ProcessMultiIndexOp::create(builder, grid, axes: gridAxes).getResults(), |
| 233 | gridAxes); |
| 234 | } |
| 235 | } // namespace mlir::shard |
| 236 | |