| 1 | //===- VectorDropLeadUnitDim.cpp - Conversion within the Vector dialect ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include <numeric> |
| 10 | |
| 11 | #include "mlir/Dialect/Utils/StructuredOpsUtils.h" |
| 12 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 13 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 14 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 15 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 16 | #include "mlir/IR/Builders.h" |
| 17 | #include "mlir/IR/TypeUtilities.h" |
| 18 | #include "llvm/ADT/STLExtras.h" |
| 19 | |
| 20 | #define DEBUG_TYPE "vector-drop-unit-dim" |
| 21 | |
| 22 | using namespace mlir; |
| 23 | using namespace mlir::vector; |
| 24 | |
| 25 | // Trims leading one dimensions (fixed-width) from `oldType` and returns the |
| 26 | // result type. Returns `vector<1xT>` if `oldType` only has one element. |
| 27 | static VectorType trimLeadingUnitDims(VectorType oldType, |
| 28 | bool trimOnlyOneDim = false, |
| 29 | bool allowRank0 = false) { |
| 30 | ArrayRef<int64_t> oldShape = oldType.getShape(); |
| 31 | ArrayRef<int64_t> newShape = oldShape; |
| 32 | |
| 33 | ArrayRef<bool> oldScalableDims = oldType.getScalableDims(); |
| 34 | ArrayRef<bool> newScalableDims = oldScalableDims; |
| 35 | |
| 36 | while (!newShape.empty() && newShape.front() == 1 && |
| 37 | !newScalableDims.front()) { |
| 38 | newShape = newShape.drop_front(N: 1); |
| 39 | newScalableDims = newScalableDims.drop_front(N: 1); |
| 40 | |
| 41 | if (trimOnlyOneDim) |
| 42 | break; |
| 43 | } |
| 44 | |
| 45 | // Make sure we have at least 1 dimension per vector type requirements. |
| 46 | if (newShape.empty() && !allowRank0) { |
| 47 | newShape = oldShape.take_back(); |
| 48 | newScalableDims = oldType.getScalableDims().take_back(); |
| 49 | } |
| 50 | return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims); |
| 51 | } |
| 52 | |
| 53 | // Returns true if one of the leading `numDims` dims of `xferOp` may be out of |
| 54 | // bounds. Dropping such a dim would make it a non-vector dim, and those must be |
| 55 | // in bounds. |
| 56 | static bool isLeadingDimOutOfBounds(VectorTransferOpInterface xferOp, |
| 57 | int64_t numDims) { |
| 58 | return llvm::any_of(Range: llvm::seq<int64_t>(Size: numDims), |
| 59 | P: [&](int64_t dim) { return !xferOp.isDimInBounds(dim); }); |
| 60 | } |
| 61 | |
| 62 | /// Return a smallVector of size `rank` containing all zeros. |
| 63 | static SmallVector<int64_t> splatZero(int64_t rank) { |
| 64 | return SmallVector<int64_t>(rank, 0); |
| 65 | } |
| 66 | |
| 67 | static Operation *createWithProperties(OpBuilder &builder, Operation *op, |
| 68 | ValueRange operands, |
| 69 | TypeRange resultTypes) { |
| 70 | OperationState state(op->getLoc(), op->getName(), operands, resultTypes, |
| 71 | op->getDiscardableAttrDictionary().getValue()); |
| 72 | state.propertiesAttr = op->getPropertiesAsAttribute(); |
| 73 | return builder.create(state); |
| 74 | } |
| 75 | namespace { |
| 76 | |
| 77 | // Casts away leading one dimensions in vector.extract_strided_slice's vector |
| 78 | // input by inserting vector.broadcast. |
| 79 | struct |
| 80 | : public OpRewritePattern<vector::ExtractStridedSliceOp> { |
| 81 | using Base::Base; |
| 82 | |
| 83 | LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp , |
| 84 | PatternRewriter &rewriter) const override { |
| 85 | // vector.extract_strided_slice requires the input and output vector to have |
| 86 | // the same rank. Here we drop leading one dimensions from the input vector |
| 87 | // type to make sure we don't cause mismatch. |
| 88 | VectorType oldSrcType = extractOp.getSourceVectorType(); |
| 89 | VectorType newSrcType = trimLeadingUnitDims(oldType: oldSrcType); |
| 90 | |
| 91 | if (newSrcType.getRank() == oldSrcType.getRank()) |
| 92 | return failure(); |
| 93 | |
| 94 | int64_t dropCount = oldSrcType.getRank() - newSrcType.getRank(); |
| 95 | |
| 96 | VectorType oldDstType = extractOp.getType(); |
| 97 | VectorType newDstType = |
| 98 | VectorType::get(shape: oldDstType.getShape().drop_front(N: dropCount), |
| 99 | elementType: oldDstType.getElementType(), |
| 100 | scalableDims: oldDstType.getScalableDims().drop_front(N: dropCount)); |
| 101 | |
| 102 | Location loc = extractOp.getLoc(); |
| 103 | |
| 104 | Value newSrcVector = rewriter.createOrFold<ShapeCastOp>( |
| 105 | location: loc, args&: newSrcType, args: extractOp.getSource()); |
| 106 | |
| 107 | // The offsets/sizes/strides attribute can have a less number of elements |
| 108 | // than the input vector's rank: it is meant for the leading dimensions. |
| 109 | auto newOffsets = rewriter.getArrayAttr( |
| 110 | value: extractOp.getOffsets().getValue().drop_front(N: dropCount)); |
| 111 | auto newSizes = rewriter.getArrayAttr( |
| 112 | value: extractOp.getSizes().getValue().drop_front(N: dropCount)); |
| 113 | auto newStrides = rewriter.getArrayAttr( |
| 114 | value: extractOp.getStrides().getValue().drop_front(N: dropCount)); |
| 115 | |
| 116 | auto = vector::ExtractStridedSliceOp::create( |
| 117 | builder&: rewriter, location: loc, resultType0: newDstType, source: newSrcVector, offsets: newOffsets, sizes: newSizes, |
| 118 | strides: newStrides); |
| 119 | |
| 120 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: oldDstType, |
| 121 | args&: newExtractOp); |
| 122 | |
| 123 | return success(); |
| 124 | } |
| 125 | }; |
| 126 | |
| 127 | // Casts away leading one dimensions in vector.insert_strided_slice's vector |
| 128 | // inputs by inserting vector.broadcast. |
| 129 | struct CastAwayInsertStridedSliceLeadingOneDim |
| 130 | : public OpRewritePattern<vector::InsertStridedSliceOp> { |
| 131 | using Base::Base; |
| 132 | |
| 133 | LogicalResult matchAndRewrite(vector::InsertStridedSliceOp insertOp, |
| 134 | PatternRewriter &rewriter) const override { |
| 135 | VectorType oldSrcType = insertOp.getSourceVectorType(); |
| 136 | VectorType newSrcType = trimLeadingUnitDims(oldType: oldSrcType); |
| 137 | VectorType oldDstType = insertOp.getDestVectorType(); |
| 138 | VectorType newDstType = trimLeadingUnitDims(oldType: oldDstType); |
| 139 | |
| 140 | int64_t srcDropCount = oldSrcType.getRank() - newSrcType.getRank(); |
| 141 | int64_t dstDropCount = oldDstType.getRank() - newDstType.getRank(); |
| 142 | if (srcDropCount == 0 && dstDropCount == 0) |
| 143 | return failure(); |
| 144 | |
| 145 | // Trim leading one dimensions from both operands. |
| 146 | Location loc = insertOp.getLoc(); |
| 147 | |
| 148 | Value newSrcVector = rewriter.createOrFold<vector::ShapeCastOp>( |
| 149 | location: loc, args&: newSrcType, args: insertOp.getValueToStore()); |
| 150 | Value newDstVector = rewriter.createOrFold<vector::ShapeCastOp>( |
| 151 | location: loc, args&: newDstType, args: insertOp.getDest()); |
| 152 | |
| 153 | auto newOffsets = rewriter.getArrayAttr( |
| 154 | value: insertOp.getOffsets().getValue().take_back(N: newDstType.getRank())); |
| 155 | auto newStrides = rewriter.getArrayAttr( |
| 156 | value: insertOp.getStrides().getValue().take_back(N: newSrcType.getRank())); |
| 157 | |
| 158 | auto newInsertOp = vector::InsertStridedSliceOp::create( |
| 159 | builder&: rewriter, location: loc, result: newDstType, valueToStore: newSrcVector, dest: newDstVector, offsets: newOffsets, |
| 160 | strides: newStrides); |
| 161 | |
| 162 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: insertOp, args&: oldDstType, |
| 163 | args&: newInsertOp); |
| 164 | |
| 165 | return success(); |
| 166 | } |
| 167 | }; |
| 168 | |
| 169 | // Casts away leading one dimensions in vector.insert's vector inputs by |
| 170 | // inserting vector.shape_cast. |
| 171 | struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> { |
| 172 | using Base::Base; |
| 173 | |
| 174 | LogicalResult matchAndRewrite(vector::InsertOp insertOp, |
| 175 | PatternRewriter &rewriter) const override { |
| 176 | Type oldSrcType = insertOp.getValueToStoreType(); |
| 177 | Type newSrcType = oldSrcType; |
| 178 | int64_t oldSrcRank = 0, newSrcRank = 0; |
| 179 | if (auto type = dyn_cast<VectorType>(Val&: oldSrcType)) { |
| 180 | newSrcType = trimLeadingUnitDims(oldType: type); |
| 181 | oldSrcRank = type.getRank(); |
| 182 | newSrcRank = cast<VectorType>(Val&: newSrcType).getRank(); |
| 183 | } |
| 184 | |
| 185 | VectorType oldDstType = insertOp.getDestVectorType(); |
| 186 | VectorType newDstType = trimLeadingUnitDims(oldType: oldDstType); |
| 187 | |
| 188 | int64_t srcDropCount = oldSrcRank - newSrcRank; |
| 189 | int64_t dstDropCount = oldDstType.getRank() - newDstType.getRank(); |
| 190 | if (srcDropCount == 0 && dstDropCount == 0) |
| 191 | return failure(); |
| 192 | |
| 193 | // Trim leading one dimensions from both operands. |
| 194 | Location loc = insertOp.getLoc(); |
| 195 | |
| 196 | Value newSrcVector = insertOp.getValueToStore(); |
| 197 | if (oldSrcRank != 0) { |
| 198 | newSrcVector = rewriter.createOrFold<vector::ShapeCastOp>( |
| 199 | location: loc, args: cast<VectorType>(Val&: newSrcType), args: insertOp.getValueToStore()); |
| 200 | } |
| 201 | Value newDstVector = rewriter.createOrFold<vector::ShapeCastOp>( |
| 202 | location: loc, args&: newDstType, args: insertOp.getDest()); |
| 203 | |
| 204 | // New position rank needs to be computed in two steps: (1) if destination |
| 205 | // type has leading unit dims, we also trim the position array accordingly, |
| 206 | // then (2) if source type also has leading unit dims, we need to append |
| 207 | // zeroes to the position array accordingly. |
| 208 | unsigned oldPosRank = insertOp.getNumIndices(); |
| 209 | unsigned newPosRank = std::max<int64_t>(a: 0, b: oldPosRank - dstDropCount); |
| 210 | SmallVector<OpFoldResult> oldPosition = insertOp.getMixedPosition(); |
| 211 | SmallVector<OpFoldResult> newPosition = |
| 212 | llvm::to_vector(Range: ArrayRef(oldPosition).take_back(N: newPosRank)); |
| 213 | newPosition.resize(N: newDstType.getRank() - newSrcRank, |
| 214 | NV: rewriter.getI64IntegerAttr(value: 0)); |
| 215 | |
| 216 | auto newInsertOp = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: newSrcVector, |
| 217 | dest: newDstVector, position: newPosition); |
| 218 | |
| 219 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: insertOp, args&: oldDstType, |
| 220 | args&: newInsertOp); |
| 221 | |
| 222 | return success(); |
| 223 | } |
| 224 | }; |
| 225 | |
| 226 | static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask, |
| 227 | VectorType newType, AffineMap newMap) { |
| 228 | VectorType newMaskType = inferTransferOpMaskType(vecType: newType, permMap: newMap); |
| 229 | |
| 230 | return vector::ShapeCastOp::create(builder&: b, location: loc, result: newMaskType, source: mask); |
| 231 | } |
| 232 | |
| 233 | // Turns vector.transfer_read on vector with leading 1 dimensions into |
| 234 | // vector.shape_cast followed by vector.transfer_read on vector without leading |
| 235 | // 1 dimensions. |
| 236 | struct CastAwayTransferReadLeadingOneDim |
| 237 | : public OpRewritePattern<vector::TransferReadOp> { |
| 238 | using Base::Base; |
| 239 | |
| 240 | LogicalResult matchAndRewrite(vector::TransferReadOp read, |
| 241 | PatternRewriter &rewriter) const override { |
| 242 | // TODO(#78787): Not supported masked op yet. |
| 243 | if (cast<MaskableOpInterface>(Val: read.getOperation()).isMasked()) |
| 244 | return failure(); |
| 245 | |
| 246 | if (read.getTransferRank() == 0) |
| 247 | return rewriter.notifyMatchFailure( |
| 248 | arg&: read, msg: "Nothing to trim - the transfer itself has rank zero" ); |
| 249 | |
| 250 | auto shapedType = cast<ShapedType>(Val: read.getBase().getType()); |
| 251 | if (shapedType.getElementType() != read.getVectorType().getElementType()) |
| 252 | return failure(); |
| 253 | |
| 254 | VectorType oldType = read.getVectorType(); |
| 255 | VectorType newType = trimLeadingUnitDims(oldType); |
| 256 | |
| 257 | if (newType == oldType) |
| 258 | return failure(); |
| 259 | |
| 260 | if (isLeadingDimOutOfBounds(xferOp: read, numDims: oldType.getRank() - newType.getRank())) |
| 261 | return rewriter.notifyMatchFailure(arg&: read, msg: "dropped dim is out of bounds" ); |
| 262 | |
| 263 | AffineMap oldMap = read.getPermutationMap(); |
| 264 | ArrayRef<AffineExpr> newResults = |
| 265 | oldMap.getResults().take_back(N: newType.getRank()); |
| 266 | AffineMap newMap = |
| 267 | AffineMap::get(dimCount: oldMap.getNumDims(), symbolCount: oldMap.getNumSymbols(), results: newResults, |
| 268 | context: rewriter.getContext()); |
| 269 | |
| 270 | ArrayAttr inBoundsAttr; |
| 271 | if (read.getInBounds()) |
| 272 | inBoundsAttr = rewriter.getArrayAttr( |
| 273 | value: read.getInBoundsAttr().getValue().take_back(N: newType.getRank())); |
| 274 | |
| 275 | Value mask = Value(); |
| 276 | if (read.getMask()) |
| 277 | mask = dropUnitDimsFromMask(b&: rewriter, loc: read.getLoc(), mask: read.getMask(), |
| 278 | newType, newMap); |
| 279 | |
| 280 | auto newRead = vector::TransferReadOp::create( |
| 281 | builder&: rewriter, location: read.getLoc(), result: newType, base: read.getBase(), indices: read.getIndices(), |
| 282 | permutation_map: AffineMapAttr::get(value: newMap), padding: read.getPadding(), mask, in_bounds: inBoundsAttr); |
| 283 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: read, args&: oldType, args&: newRead); |
| 284 | |
| 285 | return success(); |
| 286 | } |
| 287 | }; |
| 288 | |
| 289 | // Turns vector.transfer_write on vector with leading 1 dimensions into |
| 290 | // vector.shape_cast followed by vector.transfer_write on vector without leading |
| 291 | // 1 dimensions. |
| 292 | struct CastAwayTransferWriteLeadingOneDim |
| 293 | : public OpRewritePattern<vector::TransferWriteOp> { |
| 294 | using Base::Base; |
| 295 | |
| 296 | LogicalResult matchAndRewrite(vector::TransferWriteOp write, |
| 297 | PatternRewriter &rewriter) const override { |
| 298 | // TODO(#78787): Not supported masked op yet. |
| 299 | if (cast<MaskableOpInterface>(Val: write.getOperation()).isMasked()) |
| 300 | return failure(); |
| 301 | |
| 302 | if (write.getTransferRank() == 0) |
| 303 | return rewriter.notifyMatchFailure( |
| 304 | arg&: write, msg: "Nothing to trim - the transfer itself has rank zero" ); |
| 305 | |
| 306 | auto shapedType = dyn_cast<ShapedType>(Val: write.getBase().getType()); |
| 307 | if (shapedType.getElementType() != write.getVectorType().getElementType()) |
| 308 | return failure(); |
| 309 | |
| 310 | VectorType oldType = write.getVectorType(); |
| 311 | VectorType newType = trimLeadingUnitDims(oldType); |
| 312 | if (newType == oldType) |
| 313 | return failure(); |
| 314 | |
| 315 | if (isLeadingDimOutOfBounds(xferOp: write, numDims: oldType.getRank() - newType.getRank())) |
| 316 | return rewriter.notifyMatchFailure(arg&: write, msg: "dropped dim is out of bounds" ); |
| 317 | |
| 318 | AffineMap oldMap = write.getPermutationMap(); |
| 319 | ArrayRef<AffineExpr> newResults = |
| 320 | oldMap.getResults().take_back(N: newType.getRank()); |
| 321 | AffineMap newMap = |
| 322 | AffineMap::get(dimCount: oldMap.getNumDims(), symbolCount: oldMap.getNumSymbols(), results: newResults, |
| 323 | context: rewriter.getContext()); |
| 324 | |
| 325 | ArrayAttr inBoundsAttr; |
| 326 | if (write.getInBounds()) |
| 327 | inBoundsAttr = rewriter.getArrayAttr( |
| 328 | value: write.getInBoundsAttr().getValue().take_back(N: newType.getRank())); |
| 329 | |
| 330 | auto newVector = rewriter.createOrFold<vector::ShapeCastOp>( |
| 331 | location: write.getLoc(), args&: newType, args: write.getVector()); |
| 332 | |
| 333 | if (write.getMask()) { |
| 334 | Value newMask = dropUnitDimsFromMask(b&: rewriter, loc: write.getLoc(), |
| 335 | mask: write.getMask(), newType, newMap); |
| 336 | rewriter.replaceOpWithNewOp<vector::TransferWriteOp>( |
| 337 | op: write, args&: newVector, args: write.getBase(), args: write.getIndices(), |
| 338 | args: AffineMapAttr::get(value: newMap), args&: newMask, args&: inBoundsAttr); |
| 339 | return success(); |
| 340 | } |
| 341 | |
| 342 | rewriter.replaceOpWithNewOp<vector::TransferWriteOp>( |
| 343 | op: write, args&: newVector, args: write.getBase(), args: write.getIndices(), |
| 344 | args: AffineMapAttr::get(value: newMap), args&: inBoundsAttr); |
| 345 | return success(); |
| 346 | } |
| 347 | }; |
| 348 | |
| 349 | } // namespace |
| 350 | |
| 351 | // Takes `oldVal` and "drops" the leading unit dim with either ShapeCastOp or |
| 352 | // ExtractOp. The latter is used for rank-1 vectors to make sure that a scalar |
| 353 | // (as opposed to rank-0 vector) is generated. This is a requirement of e.g. |
| 354 | // ContractOp. |
| 355 | static Value (RewriterBase &rewriter, |
| 356 | Location loc, |
| 357 | mlir::Value oldVal) { |
| 358 | auto oldValTy = cast<VectorType>(Val: oldVal.getType()); |
| 359 | if (oldValTy.getRank() == 1) { |
| 360 | return rewriter.createOrFold<ExtractOp>(location: loc, args&: oldVal, args: 0); |
| 361 | } |
| 362 | |
| 363 | return rewriter.createOrFold<ShapeCastOp>( |
| 364 | location: loc, |
| 365 | args: trimLeadingUnitDims(oldType: oldValTy, |
| 366 | /*trimOnlyOneDim=*/true, |
| 367 | /*allowRank0=*/true), |
| 368 | args&: oldVal); |
| 369 | } |
| 370 | |
| 371 | // Takes `oldVal` and "adds" leading unit dim with either ShapeCastOp or |
| 372 | // BroadcastOp. The latter is used for scalar inputs as ShapeCastOp cannot |
| 373 | // "broadcast" from a scalar. Scalars are used (instead of rank-0 vectors) as |
| 374 | // ContractOp operands. |
| 375 | static Value restoreLeadingUnitDimViaShapeCastOrBcast(RewriterBase &rewriter, |
| 376 | Location loc, |
| 377 | mlir::Value oldVal, |
| 378 | mlir::Type newTy) { |
| 379 | if (!isa<VectorType>(Val: oldVal.getType())) { |
| 380 | return rewriter.createOrFold<BroadcastOp>(location: loc, args&: newTy, args&: oldVal); |
| 381 | } |
| 382 | |
| 383 | return rewriter.createOrFold<ShapeCastOp>(location: loc, args&: newTy, args&: oldVal); |
| 384 | } |
| 385 | |
| 386 | FailureOr<Value> |
| 387 | mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, |
| 388 | MaskingOpInterface maskingOp, |
| 389 | RewriterBase &rewriter) { |
| 390 | VectorType oldAccType = dyn_cast<VectorType>(Val: contractOp.getAccType()); |
| 391 | if (oldAccType == nullptr) |
| 392 | return failure(); |
| 393 | if (oldAccType.getRank() < 1) |
| 394 | return failure(); |
| 395 | if (oldAccType.getShape()[0] != 1 || oldAccType.getScalableDims()[0]) |
| 396 | return failure(); |
| 397 | |
| 398 | auto oldIndexingMaps = contractOp.getIndexingMapsArray(); |
| 399 | SmallVector<AffineMap> newIndexingMaps; |
| 400 | |
| 401 | auto oldIteratorTypes = contractOp.getIteratorTypes(); |
| 402 | SmallVector<Attribute> newIteratorTypes; |
| 403 | |
| 404 | // 0-th dim from the accumulator |
| 405 | int64_t dimToDrop = oldIndexingMaps[2].getDimPosition(idx: 0); |
| 406 | |
| 407 | if (!isParallelIterator(attr: oldIteratorTypes[dimToDrop])) |
| 408 | // only parallel type iterators can be dropped. |
| 409 | return failure(); |
| 410 | |
| 411 | for (const auto &it : llvm::enumerate(First&: oldIteratorTypes)) { |
| 412 | int64_t currDim = it.index(); |
| 413 | if (currDim == dimToDrop) |
| 414 | continue; |
| 415 | newIteratorTypes.push_back(Elt: it.value()); |
| 416 | } |
| 417 | |
| 418 | SmallVector<Value> operands = {contractOp.getLhs(), contractOp.getRhs(), |
| 419 | contractOp.getAcc()}; |
| 420 | SmallVector<Value> newOperands; |
| 421 | auto loc = contractOp.getLoc(); |
| 422 | |
| 423 | for (const auto &it : llvm::enumerate(First&: oldIndexingMaps)) { |
| 424 | // Check if the dim to be dropped exists as a leading dim in the operand |
| 425 | // if it does then we use vector.extract to drop it. |
| 426 | bool = false; |
| 427 | SmallVector<AffineExpr> results; |
| 428 | auto map = it.value(); |
| 429 | int64_t orginalZeroDim = it.value().getDimPosition(idx: 0); |
| 430 | if (orginalZeroDim != dimToDrop) { |
| 431 | // There are two reasons to be in this path, 1. We need to |
| 432 | // transpose the operand to make the dim to be dropped |
| 433 | // leading. 2. The dim to be dropped does not exist and in |
| 434 | // that case we dont want to add a unit transpose but we must |
| 435 | // check all the indices to make sure this is the case. |
| 436 | bool transposeNeeded = false; |
| 437 | SmallVector<int64_t> perm; |
| 438 | SmallVector<AffineExpr> transposeResults; |
| 439 | |
| 440 | for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) { |
| 441 | int64_t currDim = map.getDimPosition(idx: i); |
| 442 | if (currDim == dimToDrop) { |
| 443 | transposeNeeded = true; |
| 444 | perm.insert(I: perm.begin(), Elt: i); |
| 445 | auto targetExpr = rewriter.getAffineDimExpr(position: currDim); |
| 446 | transposeResults.insert(I: transposeResults.begin(), Elt: targetExpr); |
| 447 | } else { |
| 448 | perm.push_back(Elt: i); |
| 449 | auto targetExpr = rewriter.getAffineDimExpr(position: currDim); |
| 450 | transposeResults.push_back(Elt: targetExpr); |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | // Checks if only the outer, unit dimensions (of size 1) are permuted. |
| 455 | // Such transposes do not materially effect the underlying vector and can |
| 456 | // be omitted. EG: perm [1, 0, 2] applied to vector<1x1x8xi32> |
| 457 | bool transposeNonOuterUnitDims = false; |
| 458 | auto operandShape = cast<ShapedType>(Val: operands[it.index()].getType()); |
| 459 | for (auto [index, dim] : |
| 460 | llvm::enumerate(First: ArrayRef<int64_t>(perm).drop_back(N: 1))) { |
| 461 | if (dim != static_cast<int64_t>(index) && |
| 462 | operandShape.getDimSize(idx: index) != 1) { |
| 463 | transposeNonOuterUnitDims = true; |
| 464 | break; |
| 465 | } |
| 466 | } |
| 467 | |
| 468 | // Do the transpose now if needed so that we can drop the |
| 469 | // correct dim using extract later. |
| 470 | if (transposeNeeded) { |
| 471 | map = AffineMap::get(dimCount: map.getNumDims(), symbolCount: 0, results: transposeResults, |
| 472 | context: contractOp.getContext()); |
| 473 | if (transposeNonOuterUnitDims) { |
| 474 | // TODO: While the discussion on the validity of folding |
| 475 | // TransposeOp into ShapeCastOp continues, see e.g. |
| 476 | // * https://github.com/llvm/llvm-project/pull/219611, |
| 477 | // keep the explicit TransposeOp here. Note that existing TransposeOp |
| 478 | // folders already turn it into a ShapeCastOp, as demonstrated by the |
| 479 | // tests. Revisit this and consider inserting ShapeCastOp directly |
| 480 | // once the discussion progresses. |
| 481 | operands[it.index()] = rewriter.createOrFold<vector::TransposeOp>( |
| 482 | location: loc, args&: operands[it.index()], args&: perm); |
| 483 | } |
| 484 | } |
| 485 | } |
| 486 | // We have taken care to have the dim to be dropped be |
| 487 | // the leading dim. If its still not leading that means it |
| 488 | // does not exist in this operand and hence we do not need |
| 489 | // an extract. |
| 490 | if (map.getDimPosition(idx: 0) == dimToDrop) |
| 491 | validExtract = true; |
| 492 | |
| 493 | for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) { |
| 494 | int64_t currDim = map.getDimPosition(idx: i); |
| 495 | if (currDim == dimToDrop) |
| 496 | // This is the dim we are dropping. |
| 497 | continue; |
| 498 | auto targetExpr = rewriter.getAffineDimExpr( |
| 499 | position: currDim < dimToDrop ? currDim : currDim - 1); |
| 500 | results.push_back(Elt: targetExpr); |
| 501 | } |
| 502 | newIndexingMaps.push_back(Elt: AffineMap::get(dimCount: map.getNumDims() - 1, symbolCount: 0, results, |
| 503 | context: contractOp.getContext())); |
| 504 | // Extract if its a valid extraction, otherwise use the operand |
| 505 | // without extraction. |
| 506 | auto oldVal = operands[it.index()]; |
| 507 | newOperands.push_back( |
| 508 | Elt: validExtract |
| 509 | ? dropLeadingUnitDimViaShapeCastOrExtract(rewriter, loc, oldVal) |
| 510 | : oldVal); |
| 511 | } |
| 512 | |
| 513 | // Depending on whether this vector.contract is masked, the replacing Op |
| 514 | // should either be a new vector.contract Op or vector.mask Op. |
| 515 | Operation *newOp = vector::ContractionOp::create( |
| 516 | builder&: rewriter, location: loc, lhs: newOperands[0], rhs: newOperands[1], acc: newOperands[2], |
| 517 | indexingMaps: rewriter.getAffineMapArrayAttr(values: newIndexingMaps), |
| 518 | iteratorTypes: rewriter.getArrayAttr(value: newIteratorTypes), kind: contractOp.getKind(), |
| 519 | fastMathFlags: contractOp.getFastmath()); |
| 520 | |
| 521 | if (maskingOp) { |
| 522 | auto newMask = rewriter.createOrFold<ShapeCastOp>( |
| 523 | location: loc, |
| 524 | args: trimLeadingUnitDims(oldType: cast<VectorType>(Val: maskingOp.getMask().getType()), |
| 525 | /*trimOnlyOneDim=*/true, /*allowRank0=*/true), |
| 526 | args: maskingOp.getMask()); |
| 527 | |
| 528 | newOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: newOp, mask: newMask); |
| 529 | } |
| 530 | |
| 531 | return restoreLeadingUnitDimViaShapeCastOrBcast( |
| 532 | rewriter, loc, oldVal: newOp->getResult(idx: 0), newTy: contractOp->getResultTypes()[0]); |
| 533 | } |
| 534 | |
| 535 | namespace { |
| 536 | |
| 537 | /// Turns vector.contract on vector with leading 1 dimensions into |
| 538 | /// vector.shape_cast followed by vector.contract on vector without leading |
| 539 | /// 1 dimensions. Also performs transpose of lhs and rhs operands if required. |
| 540 | /// |
| 541 | /// TODO: While the discussion on the validity of folding TransposeOp into |
| 542 | /// ShapeCastOp continues, see e.g. |
| 543 | /// * https://github.com/llvm/llvm-project/pull/219611, |
| 544 | /// keep the explicit TransposeOp here. Once the discussion settles, revisit and |
| 545 | /// consider replacing TransposeOp with ShapeCastOp. |
| 546 | struct CastAwayContractionLeadingOneDim |
| 547 | : public MaskableOpRewritePattern<vector::ContractionOp> { |
| 548 | using MaskableOpRewritePattern::MaskableOpRewritePattern; |
| 549 | |
| 550 | FailureOr<Value> |
| 551 | matchAndRewriteMaskableOp(vector::ContractionOp contractOp, |
| 552 | MaskingOpInterface maskingOp, |
| 553 | PatternRewriter &rewriter) const override { |
| 554 | return castAwayContractionLeadingOneDim(contractOp, maskingOp, rewriter); |
| 555 | } |
| 556 | }; |
| 557 | |
| 558 | /// Looks at elementwise operations on vectors with at least one leading |
| 559 | /// dimension equal 1, e.g. vector<1x[4]x1xf32> (but not vector<2x[4]x1xf32>), |
| 560 | /// and cast aways the leading one dimensions (_plural_) and then broadcasts |
| 561 | /// the results. |
| 562 | /// |
| 563 | /// Example before: |
| 564 | /// %1 = arith.mulf %arg0, %arg1 : vector<1x4x1xf32> |
| 565 | /// Example after: |
| 566 | /// %2 = arith.mulf %0, %1 : vector<4x1xf32> |
| 567 | /// %3 = vector.broadcast %2 : vector<4x1xf32> to vector<1x4x1xf32> |
| 568 | /// |
| 569 | /// Does support scalable vectors. |
| 570 | class CastAwayElementwiseLeadingOneDim : public RewritePattern { |
| 571 | public: |
| 572 | CastAwayElementwiseLeadingOneDim(MLIRContext *context, |
| 573 | PatternBenefit benefit = 1) |
| 574 | : RewritePattern(MatchAnyOpTypeTag(), benefit, context) {} |
| 575 | |
| 576 | LogicalResult matchAndRewrite(Operation *op, |
| 577 | PatternRewriter &rewriter) const override { |
| 578 | if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1) |
| 579 | return failure(); |
| 580 | auto vecType = dyn_cast<VectorType>(Val: op->getResultTypes()[0]); |
| 581 | if (!vecType) |
| 582 | return failure(); |
| 583 | VectorType newVecType = trimLeadingUnitDims(oldType: vecType); |
| 584 | if (newVecType == vecType) |
| 585 | return failure(); |
| 586 | int64_t dropDim = vecType.getRank() - newVecType.getRank(); |
| 587 | SmallVector<Value, 4> newOperands; |
| 588 | for (Value operand : op->getOperands()) { |
| 589 | if (auto opVecType = dyn_cast<VectorType>(Val: operand.getType())) { |
| 590 | newOperands.push_back(Elt: vector::ExtractOp::create( |
| 591 | builder&: rewriter, location: op->getLoc(), source: operand, position: splatZero(rank: dropDim))); |
| 592 | } else { |
| 593 | newOperands.push_back(Elt: operand); |
| 594 | } |
| 595 | } |
| 596 | Operation *newOp = |
| 597 | createWithProperties(builder&: rewriter, op, operands: newOperands, resultTypes: TypeRange{newVecType}); |
| 598 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, args&: vecType, |
| 599 | args: newOp->getResult(idx: 0)); |
| 600 | return success(); |
| 601 | } |
| 602 | }; |
| 603 | } // namespace |
| 604 | |
| 605 | // Drops `dropDim` leading dimensions from `operand` using vector.extract when |
| 606 | // those dims are all non-scalable units (the cheap, structural rewrite); falls |
| 607 | // back to vector.shape_cast otherwise. |
| 608 | static Value dropLeadingOneDimsFromOperand(OpBuilder &b, Location loc, |
| 609 | Value operand, int64_t nDropped) { |
| 610 | auto oldType = cast<VectorType>(Val: operand.getType()); |
| 611 | ArrayRef<int64_t> leadingShape = oldType.getShape().take_front(N: nDropped); |
| 612 | ArrayRef<bool> leadingScalable = |
| 613 | oldType.getScalableDims().take_front(N: nDropped); |
| 614 | bool = |
| 615 | llvm::all_of(Range&: leadingShape, P: [](int64_t d) { return d == 1; }) && |
| 616 | llvm::none_of(Range&: leadingScalable, P: [](bool s) { return s; }); |
| 617 | if (extractable) |
| 618 | return vector::ExtractOp::create(builder&: b, location: loc, source: operand, position: splatZero(rank: nDropped)); |
| 619 | VectorType newType = VectorType::get( |
| 620 | shape: oldType.getShape().drop_front(N: nDropped), elementType: oldType.getElementType(), |
| 621 | scalableDims: oldType.getScalableDims().drop_front(N: nDropped)); |
| 622 | return vector::ShapeCastOp::create(builder&: b, location: loc, result: newType, source: operand); |
| 623 | } |
| 624 | |
| 625 | namespace { |
| 626 | |
| 627 | // Drops leading 1 dimensions from load-like memory operaitons. REmoves leading |
| 628 | // unit dimensions from the result types and then broadcasts back in those 1s, |
| 629 | // while also extracting (or shape_cast-ing) any leading unit dimensions on |
| 630 | // the input operands. |
| 631 | template <typename OpTy> |
| 632 | struct CastAwayLoadLikeLeadingOneDim : public OpRewritePattern<OpTy> { |
| 633 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 634 | |
| 635 | LogicalResult matchAndRewrite(OpTy op, |
| 636 | PatternRewriter &rewriter) const override { |
| 637 | VectorType oldResultType = op.getVectorType(); |
| 638 | VectorType newResultType = trimLeadingUnitDims(oldType: oldResultType); |
| 639 | if (newResultType == oldResultType) |
| 640 | return failure(); |
| 641 | int64_t nDropped = oldResultType.getRank() - newResultType.getRank(); |
| 642 | |
| 643 | Location loc = op.getLoc(); |
| 644 | SmallVector<Value> newOperands; |
| 645 | newOperands.reserve(N: op->getNumOperands()); |
| 646 | for (Value operand : op->getOperands()) { |
| 647 | if (isa<VectorType>(Val: operand.getType())) { |
| 648 | newOperands.push_back( |
| 649 | Elt: dropLeadingOneDimsFromOperand(b&: rewriter, loc, operand, nDropped)); |
| 650 | } else { |
| 651 | newOperands.push_back(Elt: operand); |
| 652 | } |
| 653 | } |
| 654 | |
| 655 | Operation *newOp = createWithProperties(rewriter, op, newOperands, |
| 656 | TypeRange{newResultType}); |
| 657 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, oldResultType, |
| 658 | newOp->getResult(idx: 0)); |
| 659 | return success(); |
| 660 | } |
| 661 | }; |
| 662 | |
| 663 | // Drops leading 1 dimensions from store-like memory ops. Extracts or |
| 664 | // `shape_cast`s away those leading unit dimensions and leaves any scalar |
| 665 | // operands alone. |
| 666 | template <typename OpTy> |
| 667 | struct CastAwayStoreLikeLeadingOneDim : public OpRewritePattern<OpTy> { |
| 668 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 669 | |
| 670 | LogicalResult matchAndRewrite(OpTy op, |
| 671 | PatternRewriter &rewriter) const override { |
| 672 | VectorType oldVecType = op.getVectorType(); |
| 673 | VectorType newVecType = trimLeadingUnitDims(oldType: oldVecType); |
| 674 | if (newVecType == oldVecType) |
| 675 | return failure(); |
| 676 | int64_t nDropped = oldVecType.getRank() - newVecType.getRank(); |
| 677 | |
| 678 | Location loc = op.getLoc(); |
| 679 | SmallVector<Value> newOperands; |
| 680 | newOperands.reserve(N: op->getNumOperands()); |
| 681 | for (Value operand : op->getOperands()) { |
| 682 | if (isa<VectorType>(Val: operand.getType())) { |
| 683 | newOperands.push_back( |
| 684 | Elt: dropLeadingOneDimsFromOperand(b&: rewriter, loc, operand, nDropped)); |
| 685 | } else { |
| 686 | newOperands.push_back(Elt: operand); |
| 687 | } |
| 688 | } |
| 689 | |
| 690 | Operation *newOp = |
| 691 | createWithProperties(rewriter, op, newOperands, op->getResultTypes()); |
| 692 | rewriter.replaceOp(op, newOp->getResults()); |
| 693 | return success(); |
| 694 | } |
| 695 | }; |
| 696 | |
| 697 | // Drops leading 1 dimensions from vector.constant_mask and inserts a |
| 698 | // vector.broadcast back to the original shape. |
| 699 | struct CastAwayConstantMaskLeadingOneDim |
| 700 | : public OpRewritePattern<vector::ConstantMaskOp> { |
| 701 | using Base::Base; |
| 702 | |
| 703 | LogicalResult matchAndRewrite(vector::ConstantMaskOp mask, |
| 704 | PatternRewriter &rewriter) const override { |
| 705 | VectorType oldType = mask.getType(); |
| 706 | VectorType newType = trimLeadingUnitDims(oldType); |
| 707 | |
| 708 | if (newType == oldType) |
| 709 | return failure(); |
| 710 | |
| 711 | int64_t dropDim = oldType.getRank() - newType.getRank(); |
| 712 | ArrayRef<int64_t> dimSizes = mask.getMaskDimSizes(); |
| 713 | |
| 714 | // If any of the dropped unit dims has a size of `0`, the entire mask is a |
| 715 | // zero mask, else the unit dim has no effect on the mask. |
| 716 | int64_t flatLeadingSize = |
| 717 | llvm::product_of(Range: dimSizes.take_front(N: dropDim + 1)); |
| 718 | SmallVector<int64_t> newDimSizes = {flatLeadingSize}; |
| 719 | newDimSizes.append(in_start: dimSizes.begin() + dropDim + 1, in_end: dimSizes.end()); |
| 720 | |
| 721 | auto newMask = vector::ConstantMaskOp::create(builder&: rewriter, location: mask.getLoc(), |
| 722 | resultType0: newType, mask_dim_sizes: newDimSizes); |
| 723 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: mask, args&: oldType, args&: newMask); |
| 724 | return success(); |
| 725 | } |
| 726 | }; |
| 727 | |
| 728 | } // namespace |
| 729 | |
| 730 | void mlir::vector::populateCastAwayVectorLeadingOneDimPatterns( |
| 731 | RewritePatternSet &patterns, PatternBenefit benefit) { |
| 732 | patterns |
| 733 | .add<CastAwayExtractStridedSliceLeadingOneDim, |
| 734 | CastAwayInsertStridedSliceLeadingOneDim, CastAwayInsertLeadingOneDim, |
| 735 | CastAwayConstantMaskLeadingOneDim, CastAwayTransferReadLeadingOneDim, |
| 736 | CastAwayTransferWriteLeadingOneDim, CastAwayElementwiseLeadingOneDim, |
| 737 | CastAwayContractionLeadingOneDim, |
| 738 | CastAwayLoadLikeLeadingOneDim<vector::LoadOp>, |
| 739 | CastAwayLoadLikeLeadingOneDim<vector::MaskedLoadOp>, |
| 740 | CastAwayLoadLikeLeadingOneDim<vector::ExpandLoadOp>, |
| 741 | CastAwayLoadLikeLeadingOneDim<vector::GatherOp>, |
| 742 | CastAwayStoreLikeLeadingOneDim<vector::StoreOp>, |
| 743 | CastAwayStoreLikeLeadingOneDim<vector::MaskedStoreOp>, |
| 744 | CastAwayStoreLikeLeadingOneDim<vector::CompressStoreOp>, |
| 745 | CastAwayStoreLikeLeadingOneDim<vector::ScatterOp>>( |
| 746 | arg: patterns.getContext(), args&: benefit); |
| 747 | } |
| 748 | |