| 1 | //===- VectorToSCF.cpp - Convert vector to SCF dialect ----------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements lowering of vector transfer operations to SCF. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include <numeric> |
| 14 | #include <optional> |
| 15 | |
| 16 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
| 17 | |
| 18 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 19 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 20 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 21 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 22 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 23 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 24 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 25 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 26 | #include "mlir/IR/Builders.h" |
| 27 | #include "mlir/Pass/Pass.h" |
| 28 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 29 | #include "llvm/ADT/STLExtras.h" |
| 30 | |
| 31 | namespace mlir { |
| 32 | #define GEN_PASS_DEF_CONVERTVECTORTOSCF |
| 33 | #include "mlir/Conversion/Passes.h.inc" |
| 34 | } // namespace mlir |
| 35 | |
| 36 | using namespace mlir; |
| 37 | using vector::TransferReadOp; |
| 38 | using vector::TransferWriteOp; |
| 39 | |
| 40 | namespace { |
| 41 | |
| 42 | /// Attribute name used for labeling transfer ops during progressive lowering. |
| 43 | static const char kPassLabel[] = "__vector_to_scf_lowering__" ; |
| 44 | |
| 45 | /// Return true if this transfer op operates on a source tensor. |
| 46 | static bool isTensorOp(VectorTransferOpInterface xferOp) { |
| 47 | if (isa<RankedTensorType>(Val: xferOp.getShapedType())) { |
| 48 | if (isa<vector::TransferWriteOp>(Val: xferOp)) { |
| 49 | // TransferWriteOps on tensors have a result. |
| 50 | assert(xferOp->getNumResults() > 0); |
| 51 | } |
| 52 | return true; |
| 53 | } |
| 54 | return false; |
| 55 | } |
| 56 | |
| 57 | /// Fail to match if the transfer op is masked by a vector.mask. |
| 58 | template <typename OpTy> |
| 59 | static LogicalResult checkNotMasked(OpTy xferOp, PatternRewriter &rewriter) { |
| 60 | if (xferOp.isMasked()) |
| 61 | return rewriter.notifyMatchFailure( |
| 62 | xferOp, "masked by vector.mask, lower the mask first" ); |
| 63 | return success(); |
| 64 | } |
| 65 | |
| 66 | /// Patterns that inherit from this struct have access to |
| 67 | /// VectorTransferToSCFOptions. |
| 68 | template <typename OpTy> |
| 69 | struct VectorToSCFPattern : public OpRewritePattern<OpTy> { |
| 70 | explicit VectorToSCFPattern(MLIRContext *context, |
| 71 | VectorTransferToSCFOptions opt) |
| 72 | : OpRewritePattern<OpTy>(context), options(opt) {} |
| 73 | |
| 74 | LogicalResult checkLowerTensors(VectorTransferOpInterface xferOp, |
| 75 | PatternRewriter &rewriter) const { |
| 76 | if (isTensorOp(xferOp) && !options.lowerTensors) { |
| 77 | return rewriter.notifyMatchFailure( |
| 78 | arg&: xferOp, msg: "lowering tensor transfers is disabled" ); |
| 79 | } |
| 80 | return success(); |
| 81 | } |
| 82 | |
| 83 | VectorTransferToSCFOptions options; |
| 84 | }; |
| 85 | |
| 86 | /// Given a vector transfer op, calculate which dimension of the `source` |
| 87 | /// memref should be unpacked in the next application of TransferOpConversion. |
| 88 | /// A return value of std::nullopt indicates a broadcast. |
| 89 | template <typename OpTy> |
| 90 | static std::optional<int64_t> unpackedDim(OpTy xferOp) { |
| 91 | // TODO: support 0-d corner case. |
| 92 | assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer" ); |
| 93 | auto map = xferOp.getPermutationMap(); |
| 94 | if (auto expr = dyn_cast<AffineDimExpr>(map.getResult(0))) { |
| 95 | return expr.getPosition(); |
| 96 | } |
| 97 | assert(xferOp.isBroadcastDim(0) && |
| 98 | "Expected AffineDimExpr or AffineConstantExpr" ); |
| 99 | return std::nullopt; |
| 100 | } |
| 101 | |
| 102 | /// Compute the permutation map for the new (N-1)-D vector transfer op. This |
| 103 | /// map is identical to the current permutation map, but the first result is |
| 104 | /// omitted. |
| 105 | template <typename OpTy> |
| 106 | static AffineMap unpackedPermutationMap(OpBuilder &b, OpTy xferOp) { |
| 107 | // TODO: support 0-d corner case. |
| 108 | assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer" ); |
| 109 | auto map = xferOp.getPermutationMap(); |
| 110 | return AffineMap::get(map.getNumDims(), 0, map.getResults().drop_front(), |
| 111 | b.getContext()); |
| 112 | } |
| 113 | |
| 114 | /// Calculate the indices for the new vector transfer op. |
| 115 | /// |
| 116 | /// E.g.: transfer_read %A[%a, %b, %c, %d] ... : vector<5x4x3xf32> ... |
| 117 | /// --> transfer_read %A[%a, %b + iv, %c, %d] ... vector<4x3f32> |
| 118 | /// ^^^^^^ |
| 119 | /// `iv` is the iteration variable of the (new) surrounding loop. |
| 120 | template <typename OpTy> |
| 121 | static void getXferIndices(OpBuilder &b, OpTy xferOp, Value iv, |
| 122 | SmallVector<Value, 8> &indices) { |
| 123 | typename OpTy::Adaptor adaptor(xferOp); |
| 124 | // Corresponding memref dim of the vector dim that is unpacked. |
| 125 | auto dim = unpackedDim(xferOp); |
| 126 | auto prevIndices = adaptor.getIndices(); |
| 127 | indices.append(prevIndices.begin(), prevIndices.end()); |
| 128 | |
| 129 | Location loc = xferOp.getLoc(); |
| 130 | bool isBroadcast = !dim.has_value(); |
| 131 | if (!isBroadcast) { |
| 132 | AffineExpr d0, d1; |
| 133 | bindDims(xferOp.getContext(), d0, d1); |
| 134 | Value offset = adaptor.getIndices()[*dim]; |
| 135 | indices[*dim] = |
| 136 | affine::makeComposedAffineApply(b, loc, e: d0 + d1, operands: {offset, iv}); |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | static void maybeYieldValue(OpBuilder &b, Location loc, bool hasRetVal, |
| 141 | Value value) { |
| 142 | if (hasRetVal) { |
| 143 | assert(value && "Expected non-empty value" ); |
| 144 | scf::YieldOp::create(builder&: b, location: loc, results: value); |
| 145 | } else { |
| 146 | scf::YieldOp::create(builder&: b, location: loc); |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | /// Generates a boolean Value that is true if the iv-th bit in xferOp's mask |
| 151 | /// is set to true. No such check is generated under following circumstances: |
| 152 | /// * xferOp does not have a mask. |
| 153 | /// * xferOp's mask is not 1D. (In case of (N>1)-D, a subvector of the mask is |
| 154 | /// computed and attached to the new transfer op in the pattern.) |
| 155 | /// * The to-be-unpacked dim of xferOp is a broadcast. |
| 156 | template <typename OpTy> |
| 157 | static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) { |
| 158 | if (!xferOp.getMask()) |
| 159 | return Value(); |
| 160 | if (xferOp.getMaskType().getRank() != 1) |
| 161 | return Value(); |
| 162 | if (xferOp.isBroadcastDim(0)) |
| 163 | return Value(); |
| 164 | |
| 165 | Location loc = xferOp.getLoc(); |
| 166 | return vector::ExtractOp::create(b, loc, xferOp.getMask(), iv); |
| 167 | } |
| 168 | |
| 169 | /// Helper function TransferOpConversion and TransferOp1dConversion. |
| 170 | /// Generate an in-bounds check if the transfer op may go out-of-bounds on the |
| 171 | /// specified dimension `dim` with the loop iteration variable `iv`. |
| 172 | /// E.g., when unpacking dimension 0 from: |
| 173 | /// ``` |
| 174 | /// %vec = vector.transfer_read %A[%a, %b] %cst |
| 175 | /// : vector<5x4xf32>, memref<?x?xf32> |
| 176 | /// ``` |
| 177 | /// An if check similar to this will be generated inside the loop: |
| 178 | /// ``` |
| 179 | /// %d = memref.dim %A, %c0 : memref<?x?xf32> |
| 180 | /// if (%a + iv < %d) { |
| 181 | /// (in-bounds case) |
| 182 | /// } else { |
| 183 | /// (out-of-bounds case) |
| 184 | /// } |
| 185 | /// ``` |
| 186 | /// |
| 187 | /// If the transfer is 1D and has a mask, this function generates a more complex |
| 188 | /// check also accounts for potentially masked out elements. |
| 189 | /// |
| 190 | /// This function variant returns the value returned by `inBoundsCase` or |
| 191 | /// `outOfBoundsCase`. The MLIR type of the return value must be specified in |
| 192 | /// `resultTypes`. |
| 193 | template <typename OpTy> |
| 194 | static Value generateInBoundsCheck( |
| 195 | OpBuilder &b, OpTy xferOp, Value iv, std::optional<int64_t> dim, |
| 196 | TypeRange resultTypes, |
| 197 | function_ref<Value(OpBuilder &, Location)> inBoundsCase, |
| 198 | function_ref<Value(OpBuilder &, Location)> outOfBoundsCase = nullptr) { |
| 199 | bool hasRetVal = !resultTypes.empty(); |
| 200 | Value cond; // Condition to be built... |
| 201 | |
| 202 | // Condition check 1: Access in-bounds? |
| 203 | bool isBroadcast = !dim; // No in-bounds check for broadcasts. |
| 204 | Location loc = xferOp.getLoc(); |
| 205 | ImplicitLocOpBuilder lb(xferOp.getLoc(), b); |
| 206 | if (!xferOp.isDimInBounds(0) && !isBroadcast) { |
| 207 | Value memrefDim = vector::createOrFoldDimOp(b, loc, source: xferOp.getBase(), dim: *dim); |
| 208 | AffineExpr d0, d1; |
| 209 | bindDims(xferOp.getContext(), d0, d1); |
| 210 | Value base = xferOp.getIndices()[*dim]; |
| 211 | Value memrefIdx = |
| 212 | affine::makeComposedAffineApply(b, loc, e: d0 + d1, operands: {base, iv}); |
| 213 | Value zero = arith::ConstantIndexOp::create(builder&: lb, value: 0); |
| 214 | Value nonNegative = |
| 215 | arith::CmpIOp::create(builder&: lb, predicate: arith::CmpIPredicate::sge, lhs: memrefIdx, rhs: zero); |
| 216 | Value inRange = arith::CmpIOp::create(builder&: lb, predicate: arith::CmpIPredicate::slt, |
| 217 | lhs: memrefIdx, rhs: memrefDim); |
| 218 | cond = arith::AndIOp::create(builder&: lb, lhs: nonNegative, rhs: inRange); |
| 219 | } |
| 220 | |
| 221 | // Condition check 2: Masked in? |
| 222 | if (auto maskCond = generateMaskCheck(b, xferOp, iv)) { |
| 223 | if (cond) |
| 224 | cond = arith::AndIOp::create(lb, cond, maskCond); |
| 225 | else |
| 226 | cond = maskCond; |
| 227 | } |
| 228 | |
| 229 | // If the condition is non-empty, generate an SCF::IfOp. |
| 230 | if (cond) { |
| 231 | auto check = scf::IfOp::create( |
| 232 | lb, cond, |
| 233 | /*thenBuilder=*/ |
| 234 | [&](OpBuilder &b, Location loc) { |
| 235 | maybeYieldValue(b, loc, hasRetVal, value: inBoundsCase(b, loc)); |
| 236 | }, |
| 237 | /*elseBuilder=*/ |
| 238 | [&](OpBuilder &b, Location loc) { |
| 239 | if (outOfBoundsCase) { |
| 240 | maybeYieldValue(b, loc, hasRetVal, value: outOfBoundsCase(b, loc)); |
| 241 | } else { |
| 242 | scf::YieldOp::create(builder&: b, location: loc); |
| 243 | } |
| 244 | }); |
| 245 | |
| 246 | return hasRetVal ? check.getResult(0) : Value(); |
| 247 | } |
| 248 | |
| 249 | // Condition is empty, no need for an SCF::IfOp. |
| 250 | return inBoundsCase(b, loc); |
| 251 | } |
| 252 | |
| 253 | /// In this function variant, `inBoundsCase` and `outOfBoundsCase` do not have |
| 254 | /// a return value. Consequently, this function does not have a return value. |
| 255 | template <typename OpTy> |
| 256 | static void generateInBoundsCheck( |
| 257 | OpBuilder &b, OpTy xferOp, Value iv, std::optional<int64_t> dim, |
| 258 | function_ref<void(OpBuilder &, Location)> inBoundsCase, |
| 259 | function_ref<void(OpBuilder &, Location)> outOfBoundsCase = nullptr) { |
| 260 | generateInBoundsCheck( |
| 261 | b, xferOp, iv, dim, /*resultTypes=*/TypeRange(), |
| 262 | /*inBoundsCase=*/ |
| 263 | [&](OpBuilder &b, Location loc) { |
| 264 | inBoundsCase(b, loc); |
| 265 | return Value(); |
| 266 | }, |
| 267 | /*outOfBoundsCase=*/ |
| 268 | [&](OpBuilder &b, Location loc) { |
| 269 | if (outOfBoundsCase) |
| 270 | outOfBoundsCase(b, loc); |
| 271 | return Value(); |
| 272 | }); |
| 273 | } |
| 274 | |
| 275 | /// Given an ArrayAttr, return a copy where the first element is dropped. |
| 276 | static ArrayAttr dropFirstElem(OpBuilder &b, ArrayAttr attr) { |
| 277 | if (!attr) |
| 278 | return attr; |
| 279 | return ArrayAttr::get(context: b.getContext(), value: attr.getValue().drop_front()); |
| 280 | } |
| 281 | |
| 282 | /// Add the pass label to a vector transfer op if its rank is not the target |
| 283 | /// rank. |
| 284 | template <typename OpTy> |
| 285 | static void maybeApplyPassLabel(OpBuilder &b, OpTy newXferOp, |
| 286 | unsigned targetRank) { |
| 287 | if (newXferOp.getVectorType().getRank() > targetRank) |
| 288 | newXferOp->setDiscardableAttr(kPassLabel, b.getUnitAttr()); |
| 289 | } |
| 290 | |
| 291 | namespace lowering_n_d { |
| 292 | |
| 293 | /// Helper data structure for data and mask buffers. |
| 294 | struct BufferAllocs { |
| 295 | Value dataBuffer; |
| 296 | Value maskBuffer; |
| 297 | }; |
| 298 | |
| 299 | // TODO: Parallelism and threadlocal considerations with a ParallelScope trait. |
| 300 | static Operation *getAutomaticAllocationScope(Operation *op) { |
| 301 | Operation *scope = |
| 302 | op->getParentWithTrait<OpTrait::AutomaticAllocationScope>(); |
| 303 | assert(scope && "Expected op to be inside automatic allocation scope" ); |
| 304 | return scope; |
| 305 | } |
| 306 | |
| 307 | /// Allocate temporary buffers for data (vector) and mask (if present). |
| 308 | template <typename OpTy> |
| 309 | static BufferAllocs allocBuffers(OpBuilder &b, OpTy xferOp) { |
| 310 | Location loc = xferOp.getLoc(); |
| 311 | OpBuilder::InsertionGuard guard(b); |
| 312 | Operation *scope = getAutomaticAllocationScope(xferOp); |
| 313 | assert(scope->getNumRegions() == 1 && |
| 314 | "AutomaticAllocationScope with >1 regions" ); |
| 315 | b.setInsertionPointToStart(&scope->getRegion(index: 0).front()); |
| 316 | |
| 317 | BufferAllocs result; |
| 318 | auto bufferType = MemRefType::get({}, xferOp.getVectorType()); |
| 319 | result.dataBuffer = memref::AllocaOp::create(b, loc, bufferType); |
| 320 | |
| 321 | if (xferOp.getMask()) { |
| 322 | auto maskType = MemRefType::get({}, xferOp.getMask().getType()); |
| 323 | auto maskBuffer = memref::AllocaOp::create(b, loc, maskType); |
| 324 | b.setInsertionPoint(xferOp); |
| 325 | memref::StoreOp::create(b, loc, xferOp.getMask(), maskBuffer); |
| 326 | result.maskBuffer = |
| 327 | memref::LoadOp::create(b, loc, maskBuffer, ValueRange()); |
| 328 | } |
| 329 | |
| 330 | return result; |
| 331 | } |
| 332 | |
| 333 | /// Given a MemRefType with VectorType element type, unpack one dimension from |
| 334 | /// the VectorType into the MemRefType. |
| 335 | /// |
| 336 | /// E.g.: memref<9xvector<5x6xf32>> --> memref<9x5xvector<6xf32>> |
| 337 | static FailureOr<MemRefType> unpackOneDim(MemRefType type) { |
| 338 | auto vectorType = dyn_cast<VectorType>(Val: type.getElementType()); |
| 339 | // Vectors with leading scalable dims are not supported. |
| 340 | // It may be possible to support these in future by using dynamic memref dims. |
| 341 | if (vectorType.getScalableDims().front()) |
| 342 | return failure(); |
| 343 | auto memrefShape = type.getShape(); |
| 344 | SmallVector<int64_t, 8> newMemrefShape; |
| 345 | newMemrefShape.append(in_start: memrefShape.begin(), in_end: memrefShape.end()); |
| 346 | newMemrefShape.push_back(Elt: vectorType.getDimSize(idx: 0)); |
| 347 | return MemRefType::get(shape: newMemrefShape, |
| 348 | elementType: VectorType::Builder(vectorType).dropDim(pos: 0)); |
| 349 | } |
| 350 | |
| 351 | /// Given a transfer op, find the memref from which the mask is loaded. This |
| 352 | /// is similar to Strategy<TransferWriteOp>::getBuffer. |
| 353 | template <typename OpTy> |
| 354 | static Value getMaskBuffer(OpTy xferOp) { |
| 355 | assert(xferOp.getMask() && "Expected that transfer op has mask" ); |
| 356 | auto loadOp = xferOp.getMask().template getDefiningOp<memref::LoadOp>(); |
| 357 | assert(loadOp && "Expected transfer op mask produced by LoadOp" ); |
| 358 | return loadOp.getMemRef(); |
| 359 | } |
| 360 | |
| 361 | /// Codegen strategy, depending on the operation. |
| 362 | template <typename OpTy> |
| 363 | struct Strategy; |
| 364 | |
| 365 | /// Code strategy for vector TransferReadOp. |
| 366 | template <> |
| 367 | struct Strategy<TransferReadOp> { |
| 368 | /// Find the StoreOp that is used for writing the current TransferReadOp's |
| 369 | /// result to the temporary buffer allocation. |
| 370 | static memref::StoreOp getStoreOp(TransferReadOp xferOp) { |
| 371 | assert(xferOp->hasOneUse() && "Expected exactly one use of TransferReadOp" ); |
| 372 | auto storeOp = dyn_cast<memref::StoreOp>(Val: (*xferOp->use_begin()).getOwner()); |
| 373 | assert(storeOp && "Expected TransferReadOp result used by StoreOp" ); |
| 374 | return storeOp; |
| 375 | } |
| 376 | |
| 377 | /// Find the temporary buffer allocation. All labeled TransferReadOps are |
| 378 | /// used like this, where %buf is either the buffer allocation or a type cast |
| 379 | /// of the buffer allocation: |
| 380 | /// ``` |
| 381 | /// %vec = vector.transfer_read ... { __vector_to_scf_lowering__ } ... |
| 382 | /// memref.store %vec, %buf[...] ... |
| 383 | /// ``` |
| 384 | static Value getBuffer(TransferReadOp xferOp) { |
| 385 | return getStoreOp(xferOp).getMemRef(); |
| 386 | } |
| 387 | |
| 388 | /// Retrieve the indices of the current StoreOp that stores into the buffer. |
| 389 | static void getBufferIndices(TransferReadOp xferOp, |
| 390 | SmallVector<Value, 8> &indices) { |
| 391 | auto storeOp = getStoreOp(xferOp); |
| 392 | auto prevIndices = memref::StoreOpAdaptor(storeOp).getIndices(); |
| 393 | indices.append(in_start: prevIndices.begin(), in_end: prevIndices.end()); |
| 394 | } |
| 395 | |
| 396 | /// Rewrite the TransferReadOp, assuming that there are no out-of-bounds |
| 397 | /// accesses on the to-be-unpacked dimension. |
| 398 | /// |
| 399 | /// 1. Generate a new (N-1)-d TransferReadOp using the loop iteration |
| 400 | /// variable `iv`. |
| 401 | /// 2. Store the result into the (already `vector.type_cast`ed) buffer. |
| 402 | /// |
| 403 | /// E.g.: |
| 404 | /// ``` |
| 405 | /// %vec = vector.transfer_read %A[%a+%i, %b, %c], %cst |
| 406 | /// : memref<?x?x?xf32>, vector<4x3xf32> |
| 407 | /// memref.store %vec, %buf[%i] : memref<5xvector<4x3xf32>> |
| 408 | /// ``` |
| 409 | /// Is rewritten to: |
| 410 | /// ``` |
| 411 | /// %casted = vector.type_cast %buf |
| 412 | /// : memref<5xvector<4x3xf32>> to memref<5x4xvector<3xf32>> |
| 413 | /// for %j = 0 to 4 { |
| 414 | /// %vec = vector.transfer_read %A[%a+%i, %b+%j, %c], %cst |
| 415 | /// : memref<?x?x?xf32>, vector<3xf32> |
| 416 | /// memref.store %vec, %casted[%i, %j] : memref<5x4xvector<3xf32>> |
| 417 | /// } |
| 418 | /// ``` |
| 419 | /// |
| 420 | /// Note: The loop and type cast are generated in TransferOpConversion. |
| 421 | /// The original TransferReadOp and store op are deleted in `cleanup`. |
| 422 | /// Note: The `mask` operand is set in TransferOpConversion. |
| 423 | static TransferReadOp rewriteOp(OpBuilder &b, |
| 424 | VectorTransferToSCFOptions options, |
| 425 | TransferReadOp xferOp, Value buffer, Value iv, |
| 426 | ValueRange /*loopState*/) { |
| 427 | SmallVector<Value, 8> storeIndices; |
| 428 | getBufferIndices(xferOp, indices&: storeIndices); |
| 429 | storeIndices.push_back(Elt: iv); |
| 430 | |
| 431 | SmallVector<Value, 8> xferIndices; |
| 432 | getXferIndices(b, xferOp, iv, indices&: xferIndices); |
| 433 | |
| 434 | Location loc = xferOp.getLoc(); |
| 435 | auto bufferType = dyn_cast<ShapedType>(Val: buffer.getType()); |
| 436 | auto vecType = dyn_cast<VectorType>(Val: bufferType.getElementType()); |
| 437 | auto inBoundsAttr = dropFirstElem(b, attr: xferOp.getInBoundsAttr()); |
| 438 | auto newXferOp = vector::TransferReadOp::create( |
| 439 | builder&: b, location: loc, result: vecType, base: xferOp.getBase(), indices: xferIndices, |
| 440 | permutation_map: AffineMapAttr::get(value: unpackedPermutationMap(b, xferOp)), |
| 441 | padding: xferOp.getPadding(), mask: Value(), in_bounds: inBoundsAttr); |
| 442 | |
| 443 | maybeApplyPassLabel(b, newXferOp, targetRank: options.targetRank); |
| 444 | |
| 445 | memref::StoreOp::create(builder&: b, location: loc, valueToStore: newXferOp.getVector(), memref: buffer, |
| 446 | indices: storeIndices); |
| 447 | return newXferOp; |
| 448 | } |
| 449 | |
| 450 | /// Handle out-of-bounds accesses on the to-be-unpacked dimension: Write |
| 451 | /// padding value to the temporary buffer. |
| 452 | static Value handleOutOfBoundsDim(OpBuilder &b, TransferReadOp xferOp, |
| 453 | Value buffer, Value iv, |
| 454 | ValueRange /*loopState*/) { |
| 455 | SmallVector<Value, 8> storeIndices; |
| 456 | getBufferIndices(xferOp, indices&: storeIndices); |
| 457 | storeIndices.push_back(Elt: iv); |
| 458 | |
| 459 | Location loc = xferOp.getLoc(); |
| 460 | auto bufferType = dyn_cast<ShapedType>(Val: buffer.getType()); |
| 461 | auto vecType = dyn_cast<VectorType>(Val: bufferType.getElementType()); |
| 462 | auto vec = |
| 463 | vector::BroadcastOp::create(builder&: b, location: loc, result: vecType, source: xferOp.getPadding()); |
| 464 | memref::StoreOp::create(builder&: b, location: loc, valueToStore: vec, memref: buffer, indices: storeIndices); |
| 465 | |
| 466 | return Value(); |
| 467 | } |
| 468 | |
| 469 | /// Cleanup after rewriting the op. |
| 470 | static void cleanup(PatternRewriter &rewriter, TransferReadOp xferOp, |
| 471 | scf::ForOp /*forOp*/) { |
| 472 | rewriter.eraseOp(op: getStoreOp(xferOp)); |
| 473 | rewriter.eraseOp(op: xferOp); |
| 474 | } |
| 475 | |
| 476 | /// Return the initial loop state for the generated scf.for loop. |
| 477 | static Value initialLoopState(TransferReadOp xferOp) { return Value(); } |
| 478 | }; |
| 479 | |
| 480 | /// Codegen strategy for vector TransferWriteOp. |
| 481 | template <> |
| 482 | struct Strategy<TransferWriteOp> { |
| 483 | /// Find the temporary buffer allocation. All labeled TransferWriteOps are |
| 484 | /// used like this, where %buf is either the buffer allocation or a type cast |
| 485 | /// of the buffer allocation: |
| 486 | /// ``` |
| 487 | /// %vec = memref.load %buf[...] ... |
| 488 | /// vector.transfer_write %vec ... { __vector_to_scf_lowering__ } ... |
| 489 | /// ``` |
| 490 | static Value getBuffer(TransferWriteOp xferOp) { |
| 491 | auto loadOp = xferOp.getVector().getDefiningOp<memref::LoadOp>(); |
| 492 | assert(loadOp && "Expected transfer op vector produced by LoadOp" ); |
| 493 | return loadOp.getMemRef(); |
| 494 | } |
| 495 | |
| 496 | /// Retrieve the indices of the current LoadOp that loads from the buffer. |
| 497 | static void getBufferIndices(TransferWriteOp xferOp, |
| 498 | SmallVector<Value, 8> &indices) { |
| 499 | auto loadOp = xferOp.getVector().getDefiningOp<memref::LoadOp>(); |
| 500 | auto prevIndices = memref::LoadOpAdaptor(loadOp).getIndices(); |
| 501 | indices.append(in_start: prevIndices.begin(), in_end: prevIndices.end()); |
| 502 | } |
| 503 | |
| 504 | /// Rewrite the TransferWriteOp, assuming that there are no out-of-bounds |
| 505 | /// accesses on the to-be-unpacked dimension. |
| 506 | /// |
| 507 | /// 1. Load an (N-1)-d vector from the (already `vector.type_cast`ed) buffer, |
| 508 | /// using the loop iteration variable `iv`. |
| 509 | /// 2. Generate a new (N-1)-d TransferWriteOp, writing the loaded vector back |
| 510 | /// to memory. |
| 511 | /// |
| 512 | /// Note: For more details, see comments on Strategy<TransferReadOp>. |
| 513 | static TransferWriteOp rewriteOp(OpBuilder &b, |
| 514 | VectorTransferToSCFOptions options, |
| 515 | TransferWriteOp xferOp, Value buffer, |
| 516 | Value iv, ValueRange loopState) { |
| 517 | SmallVector<Value, 8> loadIndices; |
| 518 | getBufferIndices(xferOp, indices&: loadIndices); |
| 519 | loadIndices.push_back(Elt: iv); |
| 520 | |
| 521 | SmallVector<Value, 8> xferIndices; |
| 522 | getXferIndices(b, xferOp, iv, indices&: xferIndices); |
| 523 | |
| 524 | Location loc = xferOp.getLoc(); |
| 525 | auto vec = memref::LoadOp::create(builder&: b, location: loc, memref: buffer, indices: loadIndices); |
| 526 | auto inBoundsAttr = dropFirstElem(b, attr: xferOp.getInBoundsAttr()); |
| 527 | auto source = loopState.empty() ? xferOp.getBase() : loopState[0]; |
| 528 | Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type(); |
| 529 | auto newXferOp = vector::TransferWriteOp::create( |
| 530 | builder&: b, location: loc, result: type, valueToStore: vec, base: source, indices: xferIndices, |
| 531 | permutation_map: AffineMapAttr::get(value: unpackedPermutationMap(b, xferOp)), mask: Value(), |
| 532 | in_bounds: inBoundsAttr); |
| 533 | |
| 534 | maybeApplyPassLabel(b, newXferOp, targetRank: options.targetRank); |
| 535 | |
| 536 | return newXferOp; |
| 537 | } |
| 538 | |
| 539 | /// Handle out-of-bounds accesses on the to-be-unpacked dimension. |
| 540 | static Value handleOutOfBoundsDim(OpBuilder &b, TransferWriteOp xferOp, |
| 541 | Value buffer, Value iv, |
| 542 | ValueRange loopState) { |
| 543 | return isTensorOp(xferOp) ? loopState[0] : Value(); |
| 544 | } |
| 545 | |
| 546 | /// Cleanup after rewriting the op. |
| 547 | static void cleanup(PatternRewriter &rewriter, TransferWriteOp xferOp, |
| 548 | scf::ForOp forOp) { |
| 549 | if (isTensorOp(xferOp)) { |
| 550 | assert(forOp->getNumResults() == 1 && "Expected one for loop result" ); |
| 551 | rewriter.replaceOp(op: xferOp, newValues: forOp->getResult(idx: 0)); |
| 552 | } else { |
| 553 | rewriter.eraseOp(op: xferOp); |
| 554 | } |
| 555 | } |
| 556 | |
| 557 | /// Return the initial loop state for the generated scf.for loop. |
| 558 | static Value initialLoopState(TransferWriteOp xferOp) { |
| 559 | return isTensorOp(xferOp) ? xferOp.getBase() : Value(); |
| 560 | } |
| 561 | }; |
| 562 | |
| 563 | template <typename OpTy> |
| 564 | static LogicalResult checkPrepareXferOp(OpTy xferOp, PatternRewriter &rewriter, |
| 565 | VectorTransferToSCFOptions options) { |
| 566 | if (xferOp->hasDiscardableAttr(kPassLabel)) |
| 567 | return rewriter.notifyMatchFailure( |
| 568 | xferOp, "kPassLabel is present (vector-to-scf lowering in progress)" ); |
| 569 | if (failed(checkNotMasked(xferOp, rewriter))) |
| 570 | return failure(); |
| 571 | if (xferOp.getVectorType().getRank() <= options.targetRank) |
| 572 | return rewriter.notifyMatchFailure( |
| 573 | xferOp, "xferOp vector rank <= transformation target rank" ); |
| 574 | if (xferOp.getVectorType().getScalableDims().front()) |
| 575 | return rewriter.notifyMatchFailure( |
| 576 | xferOp, "Unpacking of the leading dimension into the memref is not yet " |
| 577 | "supported for scalable dims" ); |
| 578 | if (isTensorOp(xferOp) && !options.lowerTensors) |
| 579 | return rewriter.notifyMatchFailure( |
| 580 | xferOp, "Unpacking for tensors has been disabled." ); |
| 581 | if (xferOp.getVectorType().getElementType() != |
| 582 | xferOp.getShapedType().getElementType()) |
| 583 | return rewriter.notifyMatchFailure( |
| 584 | xferOp, "Mismatching source and destination element types." ); |
| 585 | Operation *op = xferOp.getOperation(); |
| 586 | if (!op->getParentWithTrait<OpTrait::AutomaticAllocationScope>()) |
| 587 | return rewriter.notifyMatchFailure( |
| 588 | xferOp, "xferOp is not inside an automatic allocation scope" ); |
| 589 | |
| 590 | return success(); |
| 591 | } |
| 592 | |
| 593 | /// Prepare a TransferReadOp for progressive lowering. |
| 594 | /// |
| 595 | /// 1. Allocate a temporary buffer. |
| 596 | /// 2. Label the TransferReadOp, marking it eligible for progressive lowering. |
| 597 | /// 3. Store the result of the TransferReadOp into the temporary buffer. |
| 598 | /// 4. Load the result from the temporary buffer and replace all uses of the |
| 599 | /// original TransferReadOp with this load. |
| 600 | /// |
| 601 | /// E.g.: |
| 602 | /// ``` |
| 603 | /// %vec = vector.transfer_read %A[%a, %b, %c], %cst |
| 604 | /// : vector<5x4xf32>, memref<?x?x?xf32> |
| 605 | /// ``` |
| 606 | /// is rewritten to: |
| 607 | /// ``` |
| 608 | /// %0 = memref.alloca() : memref<vector<5x4xf32>> |
| 609 | /// %1 = vector.transfer_read %A[%a, %b, %c], %cst |
| 610 | /// { __vector_to_scf_lowering__ } : vector<5x4xf32>, memref<?x?x?xf32> |
| 611 | /// memref.store %1, %0[] : memref<vector<5x4xf32>> |
| 612 | /// %vec = memref.load %0[] : memref<vector<5x4xf32>> |
| 613 | /// ``` |
| 614 | /// |
| 615 | /// Note: A second temporary buffer may be allocated for the `mask` operand. |
| 616 | struct PrepareTransferReadConversion |
| 617 | : public VectorToSCFPattern<TransferReadOp> { |
| 618 | using VectorToSCFPattern<TransferReadOp>::VectorToSCFPattern; |
| 619 | |
| 620 | LogicalResult matchAndRewrite(TransferReadOp xferOp, |
| 621 | PatternRewriter &rewriter) const override { |
| 622 | if (checkPrepareXferOp(xferOp, rewriter, options).failed()) |
| 623 | return rewriter.notifyMatchFailure( |
| 624 | arg&: xferOp, msg: "checkPrepareXferOp conditions not met!" ); |
| 625 | |
| 626 | auto buffers = allocBuffers(b&: rewriter, xferOp); |
| 627 | auto *newXfer = rewriter.clone(op&: *xferOp.getOperation()); |
| 628 | newXfer->setDiscardableAttr(name: kPassLabel, value: rewriter.getUnitAttr()); |
| 629 | if (xferOp.getMask()) { |
| 630 | dyn_cast<TransferReadOp>(Val: newXfer).getMaskMutable().assign( |
| 631 | value: buffers.maskBuffer); |
| 632 | } |
| 633 | |
| 634 | Location loc = xferOp.getLoc(); |
| 635 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: newXfer->getResult(idx: 0), |
| 636 | memref: buffers.dataBuffer); |
| 637 | rewriter.replaceOpWithNewOp<memref::LoadOp>(op: xferOp, args&: buffers.dataBuffer, |
| 638 | args: ValueRange{}); |
| 639 | |
| 640 | return success(); |
| 641 | } |
| 642 | }; |
| 643 | |
| 644 | /// Prepare a TransferWriteOp for progressive lowering. |
| 645 | /// |
| 646 | /// 1. Allocate a temporary buffer. |
| 647 | /// 2. Store the vector into the buffer. |
| 648 | /// 3. Load the vector from the buffer again. |
| 649 | /// 4. Use the loaded vector as a TransferWriteOp operand and label the op, |
| 650 | /// marking it eligible for progressive lowering via TransferOpConversion. |
| 651 | /// |
| 652 | /// E.g.: |
| 653 | /// ``` |
| 654 | /// vector.transfer_write %vec, %A[%a, %b, %c] |
| 655 | /// : vector<5x4xf32>, memref<?x?x?xf32> |
| 656 | /// ``` |
| 657 | /// is rewritten to: |
| 658 | /// ``` |
| 659 | /// %0 = memref.alloca() : memref<vector<5x4xf32>> |
| 660 | /// memref.store %vec, %0[] : memref<vector<5x4xf32>> |
| 661 | /// %1 = memref.load %0[] : memref<vector<5x4xf32>> |
| 662 | /// vector.transfer_write %1, %A[%a, %b, %c] { __vector_to_scf_lowering__ } |
| 663 | /// : vector<5x4xf32>, memref<?x?x?xf32> |
| 664 | /// ``` |
| 665 | /// |
| 666 | /// Note: A second temporary buffer may be allocated for the `mask` operand. |
| 667 | struct PrepareTransferWriteConversion |
| 668 | : public VectorToSCFPattern<TransferWriteOp> { |
| 669 | using VectorToSCFPattern<TransferWriteOp>::VectorToSCFPattern; |
| 670 | |
| 671 | LogicalResult matchAndRewrite(TransferWriteOp xferOp, |
| 672 | PatternRewriter &rewriter) const override { |
| 673 | if (checkPrepareXferOp(xferOp, rewriter, options).failed()) |
| 674 | return rewriter.notifyMatchFailure( |
| 675 | arg&: xferOp, msg: "checkPrepareXferOp conditions not met!" ); |
| 676 | |
| 677 | Location loc = xferOp.getLoc(); |
| 678 | auto buffers = allocBuffers(b&: rewriter, xferOp); |
| 679 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: xferOp.getVector(), |
| 680 | memref: buffers.dataBuffer); |
| 681 | auto loadedVec = |
| 682 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: buffers.dataBuffer, indices: ValueRange{}); |
| 683 | rewriter.modifyOpInPlace(root: xferOp, callable: [&]() { |
| 684 | xferOp.getValueToStoreMutable().assign(value: loadedVec); |
| 685 | xferOp->setDiscardableAttr(name: kPassLabel, value: rewriter.getUnitAttr()); |
| 686 | }); |
| 687 | |
| 688 | if (xferOp.getMask()) { |
| 689 | rewriter.modifyOpInPlace(root: xferOp, callable: [&]() { |
| 690 | xferOp.getMaskMutable().assign(value: buffers.maskBuffer); |
| 691 | }); |
| 692 | } |
| 693 | |
| 694 | return success(); |
| 695 | } |
| 696 | }; |
| 697 | |
| 698 | /// Decompose a n-D PrintOp into a loop of elementary/scalar prints. This allows |
| 699 | /// printing both 1D scalable vectors and n-D fixed size vectors. |
| 700 | /// |
| 701 | /// E.g.: |
| 702 | /// ``` |
| 703 | /// vector.print %v : vector<[4]xi32> |
| 704 | /// ``` |
| 705 | /// is rewritten to: |
| 706 | /// ``` |
| 707 | /// %c0 = arith.constant 0 : index |
| 708 | /// %c4 = arith.constant 4 : index |
| 709 | /// %c1 = arith.constant 1 : index |
| 710 | /// %vscale = vector.vscale |
| 711 | /// %length = arith.muli %vscale, %c4 : index |
| 712 | /// %lastIndex = arith.subi %length, %c1 : index |
| 713 | /// vector.print punctuation <open> |
| 714 | /// scf.for %i = %c0 to %length step %c1 { |
| 715 | /// %el = vector.extract %v[%i] : i32 from vector<[4]xi32> |
| 716 | /// vector.print %el : i32 punctuation <no_punctuation> |
| 717 | /// %notLastIndex = arith.cmpi ult, %i, %lastIndex : index |
| 718 | /// scf.if %notLastIndex { |
| 719 | /// vector.print punctuation <comma> |
| 720 | /// } |
| 721 | /// } |
| 722 | /// vector.print punctuation <close> |
| 723 | /// vector.print |
| 724 | /// ``` |
| 725 | struct DecomposePrintOpConversion : public VectorToSCFPattern<vector::PrintOp> { |
| 726 | using VectorToSCFPattern<vector::PrintOp>::VectorToSCFPattern; |
| 727 | LogicalResult matchAndRewrite(vector::PrintOp printOp, |
| 728 | PatternRewriter &rewriter) const override { |
| 729 | if (!printOp.getSource()) |
| 730 | return failure(); |
| 731 | |
| 732 | VectorType vectorType = dyn_cast<VectorType>(Val: printOp.getPrintType()); |
| 733 | if (!vectorType) |
| 734 | return failure(); |
| 735 | |
| 736 | // Currently >= 2D scalable vectors are not supported. |
| 737 | // These can't be lowered to LLVM (as LLVM does not support scalable vectors |
| 738 | // of scalable vectors), and due to limitations of current ops can't be |
| 739 | // indexed with SSA values or flattened. This may change after |
| 740 | // https://reviews.llvm.org/D155034, though there still needs to be a path |
| 741 | // for lowering to LLVM. |
| 742 | if (vectorType.getRank() > 1 && vectorType.isScalable()) |
| 743 | return failure(); |
| 744 | |
| 745 | auto loc = printOp.getLoc(); |
| 746 | auto value = printOp.getSource(); |
| 747 | |
| 748 | if (auto intTy = dyn_cast<IntegerType>(Val: vectorType.getElementType())) { |
| 749 | // Oddly sized integers are (somewhat) buggy on a lot of backends, so to |
| 750 | // avoid issues extend them to a more standard size. |
| 751 | // https://github.com/llvm/llvm-project/issues/30613 |
| 752 | auto width = intTy.getWidth(); |
| 753 | auto legalWidth = llvm::NextPowerOf2(A: std::max(a: 8u, b: width) - 1); |
| 754 | auto legalIntTy = IntegerType::get(context: rewriter.getContext(), width: legalWidth, |
| 755 | signedness: intTy.getSignedness()); |
| 756 | // arith can only take signless integers, so we must cast back and forth. |
| 757 | auto signlessSourceVectorType = |
| 758 | vectorType.cloneWith(shape: {}, elementType: getIntTypeWithSignlessSemantics(intTy)); |
| 759 | auto signlessTargetVectorType = |
| 760 | vectorType.cloneWith(shape: {}, elementType: getIntTypeWithSignlessSemantics(intTy: legalIntTy)); |
| 761 | auto targetVectorType = vectorType.cloneWith(shape: {}, elementType: legalIntTy); |
| 762 | value = vector::BitCastOp::create(builder&: rewriter, location: loc, result: signlessSourceVectorType, |
| 763 | source: value); |
| 764 | if (value.getType() != signlessTargetVectorType) { |
| 765 | if (width == 1 || intTy.isUnsigned()) |
| 766 | value = arith::ExtUIOp::create(builder&: rewriter, location: loc, |
| 767 | out: signlessTargetVectorType, in: value); |
| 768 | else |
| 769 | value = arith::ExtSIOp::create(builder&: rewriter, location: loc, |
| 770 | out: signlessTargetVectorType, in: value); |
| 771 | } |
| 772 | value = vector::BitCastOp::create(builder&: rewriter, location: loc, result: targetVectorType, source: value); |
| 773 | vectorType = targetVectorType; |
| 774 | } |
| 775 | |
| 776 | auto scalableDimensions = vectorType.getScalableDims(); |
| 777 | auto shape = vectorType.getShape(); |
| 778 | constexpr int64_t singletonShape[] = {1}; |
| 779 | if (vectorType.getRank() == 0) |
| 780 | shape = singletonShape; |
| 781 | |
| 782 | if (vectorType.getRank() != 1) { |
| 783 | // Flatten n-D vectors to 1D. This is done to allow indexing with a |
| 784 | // non-constant value. |
| 785 | int64_t flatLength = llvm::product_of(Range&: shape); |
| 786 | auto flatVectorType = |
| 787 | VectorType::get(shape: {flatLength}, elementType: vectorType.getElementType()); |
| 788 | value = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: flatVectorType, source: value); |
| 789 | } |
| 790 | |
| 791 | vector::PrintOp firstClose; |
| 792 | SmallVector<Value, 8> loopIndices; |
| 793 | for (unsigned d = 0; d < shape.size(); d++) { |
| 794 | // Setup loop bounds and step. |
| 795 | Value lowerBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 796 | Value upperBound = |
| 797 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: shape[d]); |
| 798 | Value step = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 799 | if (!scalableDimensions.empty() && scalableDimensions[d]) { |
| 800 | auto vscale = vector::VectorScaleOp::create(builder&: rewriter, location: loc, |
| 801 | result: rewriter.getIndexType()); |
| 802 | upperBound = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: upperBound, rhs: vscale); |
| 803 | } |
| 804 | auto lastIndex = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: upperBound, rhs: step); |
| 805 | |
| 806 | // Create a loop to print the elements surrounded by parentheses. |
| 807 | vector::PrintOp::create(builder&: rewriter, location: loc, punctuation: vector::PrintPunctuation::Open); |
| 808 | auto loop = |
| 809 | scf::ForOp::create(builder&: rewriter, location: loc, lowerBound, upperBound, step); |
| 810 | auto printClose = vector::PrintOp::create( |
| 811 | builder&: rewriter, location: loc, punctuation: vector::PrintPunctuation::Close); |
| 812 | if (!firstClose) |
| 813 | firstClose = printClose; |
| 814 | |
| 815 | auto loopIdx = loop.getInductionVar(); |
| 816 | loopIndices.push_back(Elt: loopIdx); |
| 817 | |
| 818 | // Print a comma after all but the last element. |
| 819 | rewriter.setInsertionPointToStart(loop.getBody()); |
| 820 | auto notLastIndex = arith::CmpIOp::create( |
| 821 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ult, lhs: loopIdx, rhs: lastIndex); |
| 822 | scf::IfOp::create(builder&: rewriter, location: loc, cond: notLastIndex, |
| 823 | thenBuilder: [&](OpBuilder &builder, Location loc) { |
| 824 | vector::PrintOp::create( |
| 825 | builder, location: loc, punctuation: vector::PrintPunctuation::Comma); |
| 826 | scf::YieldOp::create(builder, location: loc); |
| 827 | }); |
| 828 | |
| 829 | rewriter.setInsertionPointToStart(loop.getBody()); |
| 830 | } |
| 831 | |
| 832 | // Compute the flattened index. |
| 833 | // Note: For the > rank 1 vectors this assumes non-scalable. |
| 834 | Value flatIndex; |
| 835 | auto currentStride = 1; |
| 836 | for (int d = shape.size() - 1; d >= 0; d--) { |
| 837 | auto stride = |
| 838 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: currentStride); |
| 839 | auto index = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: stride, rhs: loopIndices[d]); |
| 840 | if (flatIndex) |
| 841 | flatIndex = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: flatIndex, rhs: index); |
| 842 | else |
| 843 | flatIndex = index; |
| 844 | currentStride *= shape[d]; |
| 845 | } |
| 846 | |
| 847 | // Print the scalar elements in the inner most loop. |
| 848 | auto element = vector::ExtractOp::create(builder&: rewriter, location: loc, source: value, position: flatIndex); |
| 849 | vector::PrintOp::create(builder&: rewriter, location: loc, source: element, |
| 850 | punctuation: vector::PrintPunctuation::NoPunctuation); |
| 851 | |
| 852 | rewriter.setInsertionPointAfter(firstClose); |
| 853 | vector::PrintOp::create(builder&: rewriter, location: loc, punctuation: printOp.getPunctuation()); |
| 854 | rewriter.eraseOp(op: printOp); |
| 855 | return success(); |
| 856 | } |
| 857 | |
| 858 | static IntegerType getIntTypeWithSignlessSemantics(IntegerType intTy) { |
| 859 | return IntegerType::get(context: intTy.getContext(), width: intTy.getWidth(), |
| 860 | signedness: IntegerType::Signless); |
| 861 | }; |
| 862 | }; |
| 863 | |
| 864 | /// Progressive lowering of vector transfer ops: Unpack one dimension. |
| 865 | /// |
| 866 | /// 1. Unpack one dimension from the current buffer type and cast the buffer |
| 867 | /// to that new type. E.g.: |
| 868 | /// ``` |
| 869 | /// %vec = memref.load %0[%1] : memref<5xvector<4x3xf32>> |
| 870 | /// vector.transfer_write %vec ... |
| 871 | /// ``` |
| 872 | /// The following cast is generated: |
| 873 | /// ``` |
| 874 | /// %casted = vector.type_cast %0 |
| 875 | /// : memref<5xvector<4x3xf32>> to memref<5x4xvector<3xf32>> |
| 876 | /// ``` |
| 877 | /// 2. Generate a for loop and rewrite the transfer op according to the |
| 878 | /// corresponding Strategy<OpTy>. If the to-be-unpacked dimension can be |
| 879 | /// out-of-bounds, generate an if-check and handle both cases separately. |
| 880 | /// 3. Clean up according to the corresponding Strategy<OpTy>. |
| 881 | /// |
| 882 | /// Note: If the transfer op is a TransferWriteOp and operates on a tensor |
| 883 | /// source (as opposed to a memref source), then each iteration of the generated |
| 884 | /// scf.for loop yields the new tensor value. E.g.: |
| 885 | /// ``` |
| 886 | /// %result = scf.for i = 0 to 5 { |
| 887 | /// %0 = memref.load %buffer[i] : memref<5xvector<4x3xf32>> |
| 888 | /// %1 = vector.transfer_write %0, %source[...] |
| 889 | /// : vector<4x3xf32>, tensor<5x4x3xf32> |
| 890 | /// scf.yield %1 : tensor<5x4x3xf32> |
| 891 | /// } |
| 892 | /// ``` |
| 893 | template <typename OpTy> |
| 894 | struct TransferOpConversion : public VectorToSCFPattern<OpTy> { |
| 895 | using VectorToSCFPattern<OpTy>::VectorToSCFPattern; |
| 896 | |
| 897 | void initialize() { |
| 898 | // This pattern recursively unpacks one dimension at a time. The recursion |
| 899 | // bounded as the rank is strictly decreasing. |
| 900 | this->setHasBoundedRewriteRecursion(); |
| 901 | } |
| 902 | |
| 903 | static void getMaskBufferLoadIndices(OpTy xferOp, Value castedMaskBuffer, |
| 904 | SmallVectorImpl<Value> &loadIndices, |
| 905 | Value iv) { |
| 906 | assert(xferOp.getMask() && "Expected transfer op to have mask" ); |
| 907 | |
| 908 | // Add load indices from the previous iteration. |
| 909 | // The mask buffer depends on the permutation map, which makes determining |
| 910 | // the indices quite complex, so this is why we need to "look back" to the |
| 911 | // previous iteration to find the right indices. |
| 912 | Value maskBuffer = getMaskBuffer(xferOp); |
| 913 | for (Operation *user : maskBuffer.getUsers()) { |
| 914 | // If there is no previous load op, then the indices are empty. |
| 915 | if (auto loadOp = dyn_cast<memref::LoadOp>(Val: user)) { |
| 916 | Operation::operand_range prevIndices = loadOp.getIndices(); |
| 917 | loadIndices.append(in_start: prevIndices.begin(), in_end: prevIndices.end()); |
| 918 | break; |
| 919 | } |
| 920 | } |
| 921 | |
| 922 | // In case of broadcast: Use same indices to load from memref |
| 923 | // as before. |
| 924 | if (!xferOp.isBroadcastDim(0)) |
| 925 | loadIndices.push_back(Elt: iv); |
| 926 | } |
| 927 | |
| 928 | LogicalResult matchAndRewrite(OpTy xferOp, |
| 929 | PatternRewriter &rewriter) const override { |
| 930 | if (!xferOp->hasDiscardableAttr(kPassLabel)) |
| 931 | return rewriter.notifyMatchFailure( |
| 932 | xferOp, "kPassLabel is present (progressing lowering in progress)" ); |
| 933 | |
| 934 | // Find and cast data buffer. How the buffer can be found depends on OpTy. |
| 935 | ImplicitLocOpBuilder locB(xferOp.getLoc(), rewriter); |
| 936 | Value dataBuffer = Strategy<OpTy>::getBuffer(xferOp); |
| 937 | auto dataBufferType = dyn_cast<MemRefType>(Val: dataBuffer.getType()); |
| 938 | FailureOr<MemRefType> castedDataType = unpackOneDim(type: dataBufferType); |
| 939 | if (failed(Result: castedDataType)) |
| 940 | return rewriter.notifyMatchFailure(xferOp, |
| 941 | "Failed to unpack one vector dim." ); |
| 942 | |
| 943 | auto castedDataBuffer = |
| 944 | vector::TypeCastOp::create(builder&: locB, result: *castedDataType, memref: dataBuffer); |
| 945 | |
| 946 | // If the xferOp has a mask: Find and cast mask buffer. |
| 947 | Value castedMaskBuffer; |
| 948 | if (xferOp.getMask()) { |
| 949 | Value maskBuffer = getMaskBuffer(xferOp); |
| 950 | if (xferOp.isBroadcastDim(0) || xferOp.getMaskType().getRank() == 1) { |
| 951 | // Do not unpack a dimension of the mask, if: |
| 952 | // * To-be-unpacked transfer op dimension is a broadcast. |
| 953 | // * Mask is 1D, i.e., the mask cannot be further unpacked. |
| 954 | // (That means that all remaining dimensions of the transfer op must |
| 955 | // be broadcasted.) |
| 956 | castedMaskBuffer = maskBuffer; |
| 957 | } else { |
| 958 | // It's safe to assume the mask buffer can be unpacked if the data |
| 959 | // buffer was unpacked. |
| 960 | auto maskBufferType = cast<MemRefType>(Val: maskBuffer.getType()); |
| 961 | MemRefType castedMaskType = *unpackOneDim(type: maskBufferType); |
| 962 | castedMaskBuffer = |
| 963 | vector::TypeCastOp::create(builder&: locB, result: castedMaskType, memref: maskBuffer); |
| 964 | } |
| 965 | } |
| 966 | |
| 967 | // Loop bounds and step. |
| 968 | auto lb = arith::ConstantIndexOp::create(builder&: locB, value: 0); |
| 969 | auto ub = arith::ConstantIndexOp::create( |
| 970 | builder&: locB, value: castedDataType->getDimSize(idx: castedDataType->getRank() - 1)); |
| 971 | auto step = arith::ConstantIndexOp::create(builder&: locB, value: 1); |
| 972 | // TransferWriteOps that operate on tensors return the modified tensor and |
| 973 | // require a loop state. |
| 974 | auto loopState = Strategy<OpTy>::initialLoopState(xferOp); |
| 975 | |
| 976 | // Generate for loop. |
| 977 | auto result = scf::ForOp::create( |
| 978 | locB, lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(), |
| 979 | [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) { |
| 980 | Type stateType = loopState.empty() ? Type() : loopState[0].getType(); |
| 981 | |
| 982 | auto result = generateInBoundsCheck( |
| 983 | b, xferOp, iv, unpackedDim(xferOp), |
| 984 | stateType ? TypeRange(stateType) : TypeRange(), |
| 985 | /*inBoundsCase=*/ |
| 986 | [&](OpBuilder &b, Location loc) { |
| 987 | // Create new transfer op. |
| 988 | OpTy newXfer = Strategy<OpTy>::rewriteOp( |
| 989 | b, this->options, xferOp, castedDataBuffer, iv, loopState); |
| 990 | |
| 991 | // If old transfer op has a mask: Set mask on new transfer op. |
| 992 | // Special case: If the mask of the old transfer op is 1D and |
| 993 | // the unpacked dim is not a broadcast, no mask is needed on |
| 994 | // the new transfer op. |
| 995 | if (xferOp.getMask() && (xferOp.isBroadcastDim(0) || |
| 996 | xferOp.getMaskType().getRank() > 1)) { |
| 997 | OpBuilder::InsertionGuard guard(b); |
| 998 | b.setInsertionPoint(newXfer); // Insert load before newXfer. |
| 999 | |
| 1000 | SmallVector<Value, 8> loadIndices; |
| 1001 | getMaskBufferLoadIndices(xferOp, castedMaskBuffer, |
| 1002 | loadIndices, iv); |
| 1003 | auto mask = memref::LoadOp::create(builder&: b, location: loc, memref: castedMaskBuffer, |
| 1004 | indices: loadIndices); |
| 1005 | rewriter.modifyOpInPlace(newXfer, [&]() { |
| 1006 | newXfer.getMaskMutable().assign(mask); |
| 1007 | }); |
| 1008 | } |
| 1009 | |
| 1010 | return loopState.empty() ? Value() : newXfer->getResult(0); |
| 1011 | }, |
| 1012 | /*outOfBoundsCase=*/ |
| 1013 | [&](OpBuilder &b, Location /*loc*/) { |
| 1014 | return Strategy<OpTy>::handleOutOfBoundsDim( |
| 1015 | b, xferOp, castedDataBuffer, iv, loopState); |
| 1016 | }); |
| 1017 | |
| 1018 | maybeYieldValue(b, loc, !loopState.empty(), result); |
| 1019 | }); |
| 1020 | |
| 1021 | Strategy<OpTy>::cleanup(rewriter, xferOp, result); |
| 1022 | return success(); |
| 1023 | } |
| 1024 | }; |
| 1025 | |
| 1026 | /// Retrieves the dimensions sizes of a mask. Currently supports CreateMaskOp |
| 1027 | /// and ConstantMaskOp. |
| 1028 | template <typename VscaleConstantBuilder> |
| 1029 | static FailureOr<SmallVector<OpFoldResult>> |
| 1030 | getMaskDimSizes(Value mask, VscaleConstantBuilder &createVscaleMultiple) { |
| 1031 | if (!mask) |
| 1032 | return SmallVector<OpFoldResult>{}; |
| 1033 | if (auto createMaskOp = mask.getDefiningOp<vector::CreateMaskOp>()) { |
| 1034 | return llvm::map_to_vector(createMaskOp.getOperands(), [](Value dimSize) { |
| 1035 | return OpFoldResult(dimSize); |
| 1036 | }); |
| 1037 | } |
| 1038 | if (auto constantMask = mask.getDefiningOp<vector::ConstantMaskOp>()) { |
| 1039 | int dimIdx = 0; |
| 1040 | VectorType maskType = constantMask.getVectorType(); |
| 1041 | auto indexType = IndexType::get(context: mask.getContext()); |
| 1042 | return llvm::map_to_vector( |
| 1043 | constantMask.getMaskDimSizes(), [&](int64_t dimSize) { |
| 1044 | // A scalable dim in a constant_mask means vscale x dimSize. |
| 1045 | if (maskType.getScalableDims()[dimIdx++]) |
| 1046 | return OpFoldResult(createVscaleMultiple(dimSize)); |
| 1047 | return OpFoldResult(IntegerAttr::get(type: indexType, value: dimSize)); |
| 1048 | }); |
| 1049 | } |
| 1050 | return failure(); |
| 1051 | } |
| 1052 | |
| 1053 | /// Scalable vector lowering of transfer_write(transpose). This lowering only |
| 1054 | /// supports rank 2 (scalable) vectors, but can be used in conjunction with |
| 1055 | /// `UnrollTransferWriteConversion` to support n-D cases. The unroll conversion |
| 1056 | /// unrolls until the first scalable dimension. |
| 1057 | /// |
| 1058 | /// Example: |
| 1059 | /// |
| 1060 | /// BEFORE: |
| 1061 | /// ```mlir |
| 1062 | /// %transpose = vector.transpose %vec, [1, 0] |
| 1063 | /// : vector<4x[4]xf32> to vector<[4]x4xf32> |
| 1064 | /// vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [true, true]} |
| 1065 | /// : vector<[4]x4xf32>, memref<?x?xf32> |
| 1066 | /// ``` |
| 1067 | /// |
| 1068 | /// AFTER: |
| 1069 | /// ```mlir |
| 1070 | /// %c1 = arith.constant 1 : index |
| 1071 | /// %c4 = arith.constant 4 : index |
| 1072 | /// %c0 = arith.constant 0 : index |
| 1073 | /// %0 = vector.extract %arg0[0] : vector<[4]xf32> from vector<4x[4]xf32> |
| 1074 | /// %1 = vector.extract %arg0[1] : vector<[4]xf32> from vector<4x[4]xf32> |
| 1075 | /// %2 = vector.extract %arg0[2] : vector<[4]xf32> from vector<4x[4]xf32> |
| 1076 | /// %3 = vector.extract %arg0[3] : vector<[4]xf32> from vector<4x[4]xf32> |
| 1077 | /// %vscale = vector.vscale |
| 1078 | /// %c4_vscale = arith.muli %vscale, %c4 : index |
| 1079 | /// scf.for %idx = %c0 to %c4_vscale step %c1 { |
| 1080 | /// %4 = vector.extract %0[%idx] : f32 from vector<[4]xf32> |
| 1081 | /// %5 = vector.extract %1[%idx] : f32 from vector<[4]xf32> |
| 1082 | /// %6 = vector.extract %2[%idx] : f32 from vector<[4]xf32> |
| 1083 | /// %7 = vector.extract %3[%idx] : f32 from vector<[4]xf32> |
| 1084 | /// %slice_i = affine.apply #map(%idx)[%i] |
| 1085 | /// %slice = vector.from_elements %4, %5, %6, %7 : vector<4xf32> |
| 1086 | /// vector.transfer_write %slice, %arg1[%slice_i, %j] {in_bounds = [true]} |
| 1087 | /// : vector<4xf32>, memref<?x?xf32> |
| 1088 | /// } |
| 1089 | /// ``` |
| 1090 | struct ScalableTransposeTransferWriteConversion |
| 1091 | : VectorToSCFPattern<vector::TransferWriteOp> { |
| 1092 | using VectorToSCFPattern::VectorToSCFPattern; |
| 1093 | |
| 1094 | LogicalResult matchAndRewrite(TransferWriteOp writeOp, |
| 1095 | PatternRewriter &rewriter) const override { |
| 1096 | if (failed(Result: checkLowerTensors(xferOp: writeOp, rewriter))) |
| 1097 | return failure(); |
| 1098 | if (failed(Result: checkNotMasked(xferOp: writeOp, rewriter))) |
| 1099 | return failure(); |
| 1100 | |
| 1101 | VectorType vectorType = writeOp.getVectorType(); |
| 1102 | |
| 1103 | // Note: By comparing the scalable dims to an ArrayRef of length two this |
| 1104 | // implicitly checks the rank (is also two). |
| 1105 | ArrayRef<bool> scalableFlags = vectorType.getScalableDims(); |
| 1106 | if (scalableFlags != ArrayRef<bool>{true, false}) { |
| 1107 | return rewriter.notifyMatchFailure( |
| 1108 | arg&: writeOp, msg: "expected vector of the form vector<[N]xMxty>" ); |
| 1109 | } |
| 1110 | |
| 1111 | auto permutationMap = writeOp.getPermutationMap(); |
| 1112 | if (!permutationMap.isIdentity()) { |
| 1113 | return rewriter.notifyMatchFailure( |
| 1114 | arg&: writeOp, msg: "non-identity permutations are unsupported (lower first)" ); |
| 1115 | } |
| 1116 | |
| 1117 | // Note: This pattern is only lowering the leading dimension (to a loop), |
| 1118 | // so we only check if the leading dimension is in bounds. The in-bounds |
| 1119 | // attribute for the trailing dimension will be propagated. |
| 1120 | if (!writeOp.isDimInBounds(dim: 0)) { |
| 1121 | return rewriter.notifyMatchFailure( |
| 1122 | arg&: writeOp, msg: "out-of-bounds dims are unsupported (use masking)" ); |
| 1123 | } |
| 1124 | |
| 1125 | Value vector = writeOp.getVector(); |
| 1126 | auto transposeOp = vector.getDefiningOp<vector::TransposeOp>(); |
| 1127 | if (!transposeOp || |
| 1128 | transposeOp.getPermutation() != ArrayRef<int64_t>{1, 0}) { |
| 1129 | return rewriter.notifyMatchFailure(arg&: writeOp, msg: "source not transpose" ); |
| 1130 | } |
| 1131 | |
| 1132 | auto loc = writeOp.getLoc(); |
| 1133 | auto createVscaleMultiple = |
| 1134 | vector::makeVscaleConstantBuilder(rewriter, loc); |
| 1135 | |
| 1136 | auto maskDims = getMaskDimSizes(mask: writeOp.getMask(), createVscaleMultiple); |
| 1137 | if (failed(Result: maskDims)) { |
| 1138 | return rewriter.notifyMatchFailure(arg&: writeOp, |
| 1139 | msg: "failed to resolve mask dims" ); |
| 1140 | } |
| 1141 | |
| 1142 | int64_t fixedDimSize = vectorType.getDimSize(idx: 1); |
| 1143 | auto fixedDimOffsets = llvm::seq(Size: fixedDimSize); |
| 1144 | |
| 1145 | // Extract all slices from the source of the transpose. |
| 1146 | auto transposeSource = transposeOp.getVector(); |
| 1147 | SmallVector<Value> transposeSourceSlices = |
| 1148 | llvm::map_to_vector(C&: fixedDimOffsets, F: [&](int64_t idx) -> Value { |
| 1149 | return vector::ExtractOp::create(builder&: rewriter, location: loc, source: transposeSource, position: idx); |
| 1150 | }); |
| 1151 | |
| 1152 | // Loop bounds and step. |
| 1153 | auto lb = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1154 | auto ub = |
| 1155 | maskDims->empty() |
| 1156 | ? Value(createVscaleMultiple(vectorType.getDimSize(idx: 0))) |
| 1157 | : vector::getAsValues(builder&: rewriter, loc, foldResults: maskDims->front()).front(); |
| 1158 | auto step = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1159 | |
| 1160 | // Generate a new mask for the slice. |
| 1161 | VectorType sliceType = VectorType::Builder(vectorType).dropDim(pos: 0); |
| 1162 | Value sliceMask = nullptr; |
| 1163 | if (!maskDims->empty()) { |
| 1164 | sliceMask = vector::CreateMaskOp::create( |
| 1165 | builder&: rewriter, location: loc, type: sliceType.clone(elementType: rewriter.getI1Type()), |
| 1166 | mixedOperands: ArrayRef<OpFoldResult>(*maskDims).drop_front()); |
| 1167 | } |
| 1168 | |
| 1169 | Value initDest = isTensorOp(xferOp: writeOp) ? writeOp.getBase() : Value{}; |
| 1170 | ValueRange initLoopArgs = initDest ? initDest : ValueRange{}; |
| 1171 | auto result = scf::ForOp::create( |
| 1172 | builder&: rewriter, location: loc, lowerBound: lb, upperBound: ub, step, initArgs: initLoopArgs, |
| 1173 | odsArg4: [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) { |
| 1174 | // Indices for the new transfer op. |
| 1175 | SmallVector<Value, 8> xferIndices; |
| 1176 | getXferIndices(b, xferOp: writeOp, iv, indices&: xferIndices); |
| 1177 | |
| 1178 | // Extract a transposed slice from the source vector. |
| 1179 | SmallVector<Value> transposeElements = |
| 1180 | llvm::map_to_vector(C&: fixedDimOffsets, F: [&](int64_t idx) -> Value { |
| 1181 | return vector::ExtractOp::create( |
| 1182 | builder&: b, location: loc, source: transposeSourceSlices[idx], position: iv); |
| 1183 | }); |
| 1184 | auto sliceVec = vector::FromElementsOp::create(builder&: b, location: loc, result: sliceType, |
| 1185 | elements: transposeElements); |
| 1186 | |
| 1187 | // Create the transfer_write for the slice. |
| 1188 | Value dest = |
| 1189 | loopIterArgs.empty() ? writeOp.getBase() : loopIterArgs.front(); |
| 1190 | auto newWriteOp = vector::TransferWriteOp::create( |
| 1191 | builder&: b, location: loc, vector: sliceVec, dest, indices: xferIndices, |
| 1192 | inBounds: ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front()); |
| 1193 | if (sliceMask) |
| 1194 | newWriteOp.getMaskMutable().assign(value: sliceMask); |
| 1195 | |
| 1196 | // Yield from the loop. |
| 1197 | scf::YieldOp::create(builder&: b, location: loc, |
| 1198 | results: loopIterArgs.empty() ? ValueRange{} |
| 1199 | : newWriteOp.getResult()); |
| 1200 | }); |
| 1201 | |
| 1202 | if (isTensorOp(xferOp: writeOp)) |
| 1203 | rewriter.replaceOp(op: writeOp, newOp: result); |
| 1204 | else |
| 1205 | rewriter.eraseOp(op: writeOp); |
| 1206 | |
| 1207 | return success(); |
| 1208 | } |
| 1209 | }; |
| 1210 | |
| 1211 | } // namespace lowering_n_d |
| 1212 | |
| 1213 | namespace lowering_n_d_unrolled { |
| 1214 | |
| 1215 | /// If the original transfer op has a mask, compute the mask of the new transfer |
| 1216 | /// op (for the current iteration `i`) and assign it. |
| 1217 | template <typename OpTy> |
| 1218 | static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp, |
| 1219 | int64_t i) { |
| 1220 | if (!xferOp.getMask()) |
| 1221 | return; |
| 1222 | |
| 1223 | if (xferOp.isBroadcastDim(0)) { |
| 1224 | // To-be-unpacked dimension is a broadcast, which does not have a |
| 1225 | // corresponding mask dimension. Mask attribute remains unchanged. |
| 1226 | newXferOp.getMaskMutable().assign(xferOp.getMask()); |
| 1227 | return; |
| 1228 | } |
| 1229 | |
| 1230 | if (xferOp.getMaskType().getRank() > 1) { |
| 1231 | // Unpack one dimension of the mask. |
| 1232 | OpBuilder::InsertionGuard guard(b); |
| 1233 | b.setInsertionPoint(newXferOp); // Insert load before newXfer. |
| 1234 | |
| 1235 | llvm::SmallVector<int64_t, 1> indices({i}); |
| 1236 | Location loc = xferOp.getLoc(); |
| 1237 | auto newMask = vector::ExtractOp::create(b, loc, xferOp.getMask(), indices); |
| 1238 | newXferOp.getMaskMutable().assign(newMask); |
| 1239 | } |
| 1240 | |
| 1241 | // If we end up here: The mask of the old transfer op is 1D and the unpacked |
| 1242 | // dim is not a broadcast, so no mask is needed on the new transfer op. |
| 1243 | // `generateInBoundsCheck` will have evaluated the mask already. |
| 1244 | } |
| 1245 | |
| 1246 | /// Progressive lowering of vector TransferReadOp with unrolling: Unpack one |
| 1247 | /// dimension. This is similar to TransferOpConversion<TransferReadOp>, but no |
| 1248 | /// memref buffer is allocated and the SCF loop is fully unrolled. |
| 1249 | /// |
| 1250 | /// ``` |
| 1251 | /// E.g.: |
| 1252 | /// ``` |
| 1253 | /// %vec = vector.transfer_read %A[%a, %b, %c], %padding |
| 1254 | /// : memref<?x?x?xf32>, vector<5x4xf32> |
| 1255 | /// ``` |
| 1256 | /// is rewritten to IR such as (simplified): |
| 1257 | /// ``` |
| 1258 | /// %v_init = splat %padding : vector<5x4xf32> |
| 1259 | /// %tmp0 = vector.transfer_read %A[%a, %b, %c], %padding |
| 1260 | /// : memref<?x?x?xf32>, vector<4xf32> |
| 1261 | /// %v0 = vector.insert %tmp0, %v_init[0] : vector<4xf32> into vector<5x4xf32> |
| 1262 | /// %tmp1 = vector.transfer_read %A[%a, %b + 1, %c], %padding |
| 1263 | /// : memref<?x?x?xf32>, vector<4xf32> |
| 1264 | /// %v1 = vector.insert %tmp1, %v0[1] : vector<4xf32> into vector<5x4xf32> |
| 1265 | /// ... |
| 1266 | /// %tmp4 = vector.transfer_read %A[%a, %b + 4, %c], %padding |
| 1267 | /// : memref<?x?x?xf32>, vector<4xf32> |
| 1268 | /// %vec = vector.insert %tmp1, %v3[4] : vector<4xf32> into vector<5x4xf32> |
| 1269 | /// ``` |
| 1270 | /// |
| 1271 | /// Note: As an optimization, if the result of the original TransferReadOp |
| 1272 | /// was directly inserted into another vector, no new %v_init vector is created. |
| 1273 | /// Instead, the new TransferReadOp results are inserted into that vector. |
| 1274 | struct UnrollTransferReadConversion |
| 1275 | : public VectorToSCFPattern<TransferReadOp> { |
| 1276 | using VectorToSCFPattern<TransferReadOp>::VectorToSCFPattern; |
| 1277 | |
| 1278 | void initialize() { |
| 1279 | // This pattern recursively unpacks one dimension at a time. The recursion |
| 1280 | // bounded as the rank is strictly decreasing. |
| 1281 | setHasBoundedRewriteRecursion(); |
| 1282 | } |
| 1283 | |
| 1284 | /// Get or build the vector into which the newly created TransferReadOp |
| 1285 | /// results are inserted. |
| 1286 | Value buildResultVector(PatternRewriter &rewriter, |
| 1287 | TransferReadOp xferOp) const { |
| 1288 | if (auto insertOp = getInsertOp(xferOp)) |
| 1289 | return insertOp.getDest(); |
| 1290 | Location loc = xferOp.getLoc(); |
| 1291 | return vector::BroadcastOp::create(builder&: rewriter, location: loc, result: xferOp.getVectorType(), |
| 1292 | source: xferOp.getPadding()); |
| 1293 | } |
| 1294 | |
| 1295 | /// If the result of the TransferReadOp has exactly one user, which is a |
| 1296 | /// vector::InsertOp, return that operation. |
| 1297 | vector::InsertOp getInsertOp(TransferReadOp xferOp) const { |
| 1298 | if (xferOp->hasOneUse()) { |
| 1299 | Operation *xferOpUser = *xferOp->getUsers().begin(); |
| 1300 | if (auto insertOp = dyn_cast<vector::InsertOp>(Val: xferOpUser)) |
| 1301 | return insertOp; |
| 1302 | } |
| 1303 | |
| 1304 | return vector::InsertOp(); |
| 1305 | } |
| 1306 | |
| 1307 | /// If the result of the TransferReadOp has exactly one user, which is a |
| 1308 | /// vector::InsertOp, return that operation's indices. |
| 1309 | void getInsertionIndices(TransferReadOp xferOp, |
| 1310 | SmallVectorImpl<OpFoldResult> &indices) const { |
| 1311 | if (auto insertOp = getInsertOp(xferOp)) { |
| 1312 | auto pos = insertOp.getMixedPosition(); |
| 1313 | indices.append(in_start: pos.begin(), in_end: pos.end()); |
| 1314 | } |
| 1315 | } |
| 1316 | |
| 1317 | /// Rewrite the op: Unpack one dimension. Can handle masks, out-of-bounds |
| 1318 | /// accesses, and broadcasts and transposes in permutation maps. |
| 1319 | LogicalResult matchAndRewrite(TransferReadOp xferOp, |
| 1320 | PatternRewriter &rewriter) const override { |
| 1321 | if (xferOp.getVectorType().getRank() <= options.targetRank) |
| 1322 | return rewriter.notifyMatchFailure( |
| 1323 | arg&: xferOp, msg: "vector rank is less or equal to target rank" ); |
| 1324 | if (failed(Result: checkLowerTensors(xferOp, rewriter))) |
| 1325 | return failure(); |
| 1326 | if (failed(Result: checkNotMasked(xferOp, rewriter))) |
| 1327 | return failure(); |
| 1328 | if (xferOp.getVectorType().getElementType() != |
| 1329 | xferOp.getShapedType().getElementType()) |
| 1330 | return rewriter.notifyMatchFailure( |
| 1331 | arg&: xferOp, msg: "not yet supported: element type mismatch" ); |
| 1332 | auto xferVecType = xferOp.getVectorType(); |
| 1333 | if (xferVecType.getScalableDims()[0]) { |
| 1334 | return rewriter.notifyMatchFailure( |
| 1335 | arg&: xferOp, msg: "scalable dimensions cannot be unrolled at compile time" ); |
| 1336 | } |
| 1337 | |
| 1338 | auto insertOp = getInsertOp(xferOp); |
| 1339 | auto vec = buildResultVector(rewriter, xferOp); |
| 1340 | auto vecType = dyn_cast<VectorType>(Val: vec.getType()); |
| 1341 | |
| 1342 | VectorType newXferVecType = VectorType::Builder(xferVecType).dropDim(pos: 0); |
| 1343 | |
| 1344 | int64_t dimSize = xferVecType.getShape()[0]; |
| 1345 | |
| 1346 | // Generate fully unrolled loop of transfer ops. |
| 1347 | Location loc = xferOp.getLoc(); |
| 1348 | for (int64_t i = 0; i < dimSize; ++i) { |
| 1349 | Value iv = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i); |
| 1350 | |
| 1351 | // FIXME: Rename this lambda - it does much more than just |
| 1352 | // in-bounds-check generation. |
| 1353 | vec = generateInBoundsCheck( |
| 1354 | b&: rewriter, xferOp, iv, dim: unpackedDim(xferOp), resultTypes: TypeRange(vecType), |
| 1355 | /*inBoundsCase=*/ |
| 1356 | [&](OpBuilder &b, Location loc) { |
| 1357 | // Indices for the new transfer op. |
| 1358 | SmallVector<Value, 8> xferIndices; |
| 1359 | getXferIndices(b, xferOp, iv, indices&: xferIndices); |
| 1360 | |
| 1361 | // Indices for the new vector.insert op. |
| 1362 | SmallVector<OpFoldResult, 8> insertionIndices; |
| 1363 | getInsertionIndices(xferOp, indices&: insertionIndices); |
| 1364 | insertionIndices.push_back(Elt: rewriter.getIndexAttr(value: i)); |
| 1365 | |
| 1366 | auto inBoundsAttr = dropFirstElem(b, attr: xferOp.getInBoundsAttr()); |
| 1367 | |
| 1368 | auto newXferOp = vector::TransferReadOp::create( |
| 1369 | builder&: b, location: loc, result: newXferVecType, base: xferOp.getBase(), indices: xferIndices, |
| 1370 | permutation_map: AffineMapAttr::get(value: unpackedPermutationMap(b, xferOp)), |
| 1371 | padding: xferOp.getPadding(), mask: Value(), in_bounds: inBoundsAttr); |
| 1372 | maybeAssignMask(b, xferOp, newXferOp, i); |
| 1373 | |
| 1374 | Value valToInser = newXferOp.getResult(); |
| 1375 | if (newXferVecType.getRank() == 0) { |
| 1376 | // vector.insert does not accept rank-0 as the non-indexed |
| 1377 | // argument. Extract the scalar before inserting. |
| 1378 | valToInser = vector::ExtractOp::create(builder&: b, location: loc, source: valToInser, |
| 1379 | position: SmallVector<int64_t>()); |
| 1380 | } |
| 1381 | return vector::InsertOp::create(builder&: b, location: loc, valueToStore: valToInser, dest: vec, |
| 1382 | position: insertionIndices); |
| 1383 | }, |
| 1384 | /*outOfBoundsCase=*/ |
| 1385 | [&](OpBuilder &b, Location loc) { |
| 1386 | // Loop through original (unmodified) vector. |
| 1387 | return vec; |
| 1388 | }); |
| 1389 | } |
| 1390 | |
| 1391 | if (insertOp) { |
| 1392 | // Rewrite single user of the old TransferReadOp, which was an InsertOp. |
| 1393 | rewriter.replaceOp(op: insertOp, newValues: vec); |
| 1394 | rewriter.eraseOp(op: xferOp); |
| 1395 | } else { |
| 1396 | rewriter.replaceOp(op: xferOp, newValues: vec); |
| 1397 | } |
| 1398 | |
| 1399 | return success(); |
| 1400 | } |
| 1401 | }; |
| 1402 | |
| 1403 | /// Progressive lowering of vector TransferWriteOp with unrolling: Unpack one |
| 1404 | /// dimension. This is similar to TransferOpConversion<TransferWriteOp>, but no |
| 1405 | /// memref buffer is allocated and the SCF loop is fully unrolled. |
| 1406 | /// |
| 1407 | /// ``` |
| 1408 | /// E.g.: |
| 1409 | /// ``` |
| 1410 | /// vector.transfer_write %vec, %A[%a, %b, %c] |
| 1411 | /// : vector<5x4xf32>, memref<?x?x?xf32> |
| 1412 | /// ``` |
| 1413 | /// is rewritten to IR such as (simplified): |
| 1414 | /// ``` |
| 1415 | /// %v0 = vector.extract %vec[0] : vector<4xf32> from vector<5x4xf32> |
| 1416 | /// vector.transfer_write %v0, %A[%a, %b, %c] : vector<4xf32>, memref<...> |
| 1417 | /// %v1 = vector.extract %vec[1] : vector<4xf32> from vector<5x4xf32> |
| 1418 | /// vector.transfer_write %v1, %A[%a, %b + 1, %c] : vector<4xf32>, memref<...> |
| 1419 | /// ... |
| 1420 | /// %v4 = vector.extract %vec[4] : vector<4xf32> from vector<5x4xf32> |
| 1421 | /// vector.transfer_write %v4, %A[%a, %b + 4, %c] : vector<4xf32>, memref<...> |
| 1422 | /// ``` |
| 1423 | /// |
| 1424 | /// Note: As an optimization, if the vector of the original TransferWriteOp |
| 1425 | /// was directly extracted from another vector via an ExtractOp `a`, extract |
| 1426 | /// the vectors for the newly generated TransferWriteOps from `a`'s input. By |
| 1427 | /// doing so, `a` may become dead, and the number of ExtractOps generated during |
| 1428 | /// recursive application of this pattern will be minimal. |
| 1429 | struct UnrollTransferWriteConversion |
| 1430 | : public VectorToSCFPattern<TransferWriteOp> { |
| 1431 | using VectorToSCFPattern<TransferWriteOp>::VectorToSCFPattern; |
| 1432 | |
| 1433 | void initialize() { |
| 1434 | // This pattern recursively unpacks one dimension at a time. The recursion |
| 1435 | // bounded as the rank is strictly decreasing. |
| 1436 | setHasBoundedRewriteRecursion(); |
| 1437 | } |
| 1438 | |
| 1439 | /// Return the vector from which newly generated ExtracOps will extract. |
| 1440 | Value getDataVector(TransferWriteOp xferOp) const { |
| 1441 | if (auto = getExtractOp(xferOp)) |
| 1442 | return extractOp.getSource(); |
| 1443 | return xferOp.getVector(); |
| 1444 | } |
| 1445 | |
| 1446 | /// If the input of the given TransferWriteOp is an ExtractOp, return it. |
| 1447 | vector::ExtractOp (TransferWriteOp xferOp) const { |
| 1448 | if (auto *op = xferOp.getVector().getDefiningOp()) |
| 1449 | return dyn_cast<vector::ExtractOp>(Val: op); |
| 1450 | return vector::ExtractOp(); |
| 1451 | } |
| 1452 | |
| 1453 | /// If the input of the given TransferWriteOp is an ExtractOp, return its |
| 1454 | /// indices. |
| 1455 | void (TransferWriteOp xferOp, |
| 1456 | SmallVectorImpl<OpFoldResult> &indices) const { |
| 1457 | if (auto = getExtractOp(xferOp)) { |
| 1458 | auto pos = extractOp.getMixedPosition(); |
| 1459 | indices.append(in_start: pos.begin(), in_end: pos.end()); |
| 1460 | } |
| 1461 | } |
| 1462 | |
| 1463 | /// Rewrite the op: Unpack one dimension. Can handle masks, out-of-bounds |
| 1464 | /// accesses, and broadcasts and transposes in permutation maps. |
| 1465 | LogicalResult matchAndRewrite(TransferWriteOp xferOp, |
| 1466 | PatternRewriter &rewriter) const override { |
| 1467 | VectorType inputVectorTy = xferOp.getVectorType(); |
| 1468 | |
| 1469 | if (inputVectorTy.getRank() <= options.targetRank) |
| 1470 | return failure(); |
| 1471 | |
| 1472 | if (failed(Result: checkLowerTensors(xferOp, rewriter))) |
| 1473 | return failure(); |
| 1474 | if (failed(Result: checkNotMasked(xferOp, rewriter))) |
| 1475 | return failure(); |
| 1476 | // Transfer ops that modify the element type are not supported atm. |
| 1477 | if (inputVectorTy.getElementType() != |
| 1478 | xferOp.getShapedType().getElementType()) |
| 1479 | return failure(); |
| 1480 | |
| 1481 | auto vec = getDataVector(xferOp); |
| 1482 | if (inputVectorTy.getScalableDims()[0]) { |
| 1483 | // Cannot unroll a scalable dimension at compile time. |
| 1484 | return failure(); |
| 1485 | } |
| 1486 | |
| 1487 | int64_t dimSize = inputVectorTy.getShape()[0]; |
| 1488 | Value source = xferOp.getBase(); // memref or tensor to be written to. |
| 1489 | auto sourceType = isTensorOp(xferOp) ? xferOp.getShapedType() : Type(); |
| 1490 | |
| 1491 | // Generate fully unrolled loop of transfer ops. |
| 1492 | Location loc = xferOp.getLoc(); |
| 1493 | for (int64_t i = 0; i < dimSize; ++i) { |
| 1494 | Value iv = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i); |
| 1495 | |
| 1496 | auto updatedSource = generateInBoundsCheck( |
| 1497 | b&: rewriter, xferOp, iv, dim: unpackedDim(xferOp), |
| 1498 | resultTypes: isTensorOp(xferOp) ? TypeRange(sourceType) : TypeRange(), |
| 1499 | /*inBoundsCase=*/ |
| 1500 | [&](OpBuilder &b, Location loc) { |
| 1501 | // Indices for the new transfer op. |
| 1502 | SmallVector<Value, 8> xferIndices; |
| 1503 | getXferIndices(b, xferOp, iv, indices&: xferIndices); |
| 1504 | |
| 1505 | // Indices for the new vector.extract op. |
| 1506 | SmallVector<OpFoldResult, 8> ; |
| 1507 | getExtractionIndices(xferOp, indices&: extractionIndices); |
| 1508 | extractionIndices.push_back(Elt: b.getI64IntegerAttr(value: i)); |
| 1509 | |
| 1510 | auto = |
| 1511 | vector::ExtractOp::create(builder&: b, location: loc, source: vec, position: extractionIndices); |
| 1512 | auto inBoundsAttr = dropFirstElem(b, attr: xferOp.getInBoundsAttr()); |
| 1513 | Value xferVec; |
| 1514 | if (inputVectorTy.getRank() == 1) { |
| 1515 | // When target-rank=0, unrolling would causes the vector input |
| 1516 | // argument into `transfer_write` to become a scalar. We solve |
| 1517 | // this by broadcasting the scalar to a 0D vector. |
| 1518 | xferVec = vector::BroadcastOp::create( |
| 1519 | builder&: b, location: loc, result: VectorType::get(shape: {}, elementType: extracted.getType()), source: extracted); |
| 1520 | } else { |
| 1521 | xferVec = extracted; |
| 1522 | } |
| 1523 | auto newXferOp = vector::TransferWriteOp::create( |
| 1524 | builder&: b, location: loc, result: sourceType, valueToStore: xferVec, base: source, indices: xferIndices, |
| 1525 | permutation_map: AffineMapAttr::get(value: unpackedPermutationMap(b, xferOp)), mask: Value(), |
| 1526 | in_bounds: inBoundsAttr); |
| 1527 | |
| 1528 | maybeAssignMask(b, xferOp, newXferOp, i); |
| 1529 | |
| 1530 | return isTensorOp(xferOp) ? newXferOp->getResult(idx: 0) : Value(); |
| 1531 | }, |
| 1532 | /*outOfBoundsCase=*/ |
| 1533 | [&](OpBuilder &b, Location loc) { |
| 1534 | return isTensorOp(xferOp) ? source : Value(); |
| 1535 | }); |
| 1536 | |
| 1537 | if (isTensorOp(xferOp)) |
| 1538 | source = updatedSource; |
| 1539 | } |
| 1540 | |
| 1541 | if (isTensorOp(xferOp)) |
| 1542 | rewriter.replaceOp(op: xferOp, newValues: source); |
| 1543 | else |
| 1544 | rewriter.eraseOp(op: xferOp); |
| 1545 | |
| 1546 | return success(); |
| 1547 | } |
| 1548 | }; |
| 1549 | |
| 1550 | } // namespace lowering_n_d_unrolled |
| 1551 | |
| 1552 | namespace lowering_1_d { |
| 1553 | |
| 1554 | /// Compute the indices into the memref for the LoadOp/StoreOp generated as |
| 1555 | /// part of TransferOp1dConversion. Return the memref dimension on which |
| 1556 | /// the transfer is operating. A return value of std::nullopt indicates a |
| 1557 | /// broadcast. |
| 1558 | template <typename OpTy> |
| 1559 | static std::optional<int64_t> |
| 1560 | get1dMemrefIndices(OpBuilder &b, OpTy xferOp, Value iv, |
| 1561 | SmallVector<Value, 8> &memrefIndices) { |
| 1562 | auto indices = xferOp.getIndices(); |
| 1563 | auto map = xferOp.getPermutationMap(); |
| 1564 | assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer" ); |
| 1565 | |
| 1566 | memrefIndices.append(indices.begin(), indices.end()); |
| 1567 | assert(map.getNumResults() == 1 && |
| 1568 | "Expected 1 permutation map result for 1D transfer" ); |
| 1569 | if (auto expr = dyn_cast<AffineDimExpr>(map.getResult(0))) { |
| 1570 | Location loc = xferOp.getLoc(); |
| 1571 | auto dim = expr.getPosition(); |
| 1572 | AffineExpr d0, d1; |
| 1573 | bindDims(xferOp.getContext(), d0, d1); |
| 1574 | Value offset = memrefIndices[dim]; |
| 1575 | memrefIndices[dim] = |
| 1576 | affine::makeComposedAffineApply(b, loc, e: d0 + d1, operands: {offset, iv}); |
| 1577 | return dim; |
| 1578 | } |
| 1579 | |
| 1580 | assert(xferOp.isBroadcastDim(0) && |
| 1581 | "Expected AffineDimExpr or AffineConstantExpr" ); |
| 1582 | return std::nullopt; |
| 1583 | } |
| 1584 | |
| 1585 | /// Codegen strategy for TransferOp1dConversion, depending on the |
| 1586 | /// operation. |
| 1587 | template <typename OpTy> |
| 1588 | struct Strategy1d; |
| 1589 | |
| 1590 | /// Codegen strategy for TransferReadOp. |
| 1591 | template <> |
| 1592 | struct Strategy1d<TransferReadOp> { |
| 1593 | static void generateForLoopBody(OpBuilder &b, Location loc, |
| 1594 | TransferReadOp xferOp, Value iv, |
| 1595 | ValueRange loopState) { |
| 1596 | SmallVector<Value, 8> indices; |
| 1597 | auto dim = get1dMemrefIndices(b, xferOp, iv, memrefIndices&: indices); |
| 1598 | auto vec = loopState[0]; |
| 1599 | |
| 1600 | // In case of out-of-bounds access, leave `vec` as is (was initialized with |
| 1601 | // padding value). |
| 1602 | auto nextVec = generateInBoundsCheck( |
| 1603 | b, xferOp, iv, dim, resultTypes: TypeRange(xferOp.getVectorType()), |
| 1604 | /*inBoundsCase=*/ |
| 1605 | [&](OpBuilder &b, Location loc) { |
| 1606 | Value val = memref::LoadOp::create(builder&: b, location: loc, memref: xferOp.getBase(), indices); |
| 1607 | return vector::InsertOp::create(builder&: b, location: loc, valueToStore: val, dest: vec, position: iv); |
| 1608 | }, |
| 1609 | /*outOfBoundsCase=*/ |
| 1610 | [&](OpBuilder & /*b*/, Location loc) { return vec; }); |
| 1611 | scf::YieldOp::create(builder&: b, location: loc, results: nextVec); |
| 1612 | } |
| 1613 | |
| 1614 | static Value initialLoopState(OpBuilder &b, TransferReadOp xferOp) { |
| 1615 | // Inititalize vector with padding value. |
| 1616 | Location loc = xferOp.getLoc(); |
| 1617 | return vector::BroadcastOp::create(builder&: b, location: loc, result: xferOp.getVectorType(), |
| 1618 | source: xferOp.getPadding()); |
| 1619 | } |
| 1620 | }; |
| 1621 | |
| 1622 | /// Codegen strategy for TransferWriteOp. |
| 1623 | template <> |
| 1624 | struct Strategy1d<TransferWriteOp> { |
| 1625 | static void generateForLoopBody(OpBuilder &b, Location loc, |
| 1626 | TransferWriteOp xferOp, Value iv, |
| 1627 | ValueRange /*loopState*/) { |
| 1628 | SmallVector<Value, 8> indices; |
| 1629 | auto dim = get1dMemrefIndices(b, xferOp, iv, memrefIndices&: indices); |
| 1630 | |
| 1631 | // Nothing to do in case of out-of-bounds access. |
| 1632 | generateInBoundsCheck( |
| 1633 | b, xferOp, iv, dim, |
| 1634 | /*inBoundsCase=*/[&](OpBuilder &b, Location loc) { |
| 1635 | auto val = vector::ExtractOp::create(builder&: b, location: loc, source: xferOp.getVector(), position: iv); |
| 1636 | memref::StoreOp::create(builder&: b, location: loc, valueToStore: val, memref: xferOp.getBase(), indices); |
| 1637 | }); |
| 1638 | scf::YieldOp::create(builder&: b, location: loc); |
| 1639 | } |
| 1640 | |
| 1641 | static Value initialLoopState(OpBuilder &b, TransferWriteOp xferOp) { |
| 1642 | return Value(); |
| 1643 | } |
| 1644 | }; |
| 1645 | |
| 1646 | /// Lower a 1D vector transfer op to SCF using scalar loads/stores. This is |
| 1647 | /// necessary in cases where a 1D vector transfer op cannot be lowered into |
| 1648 | /// vector load/stores due to non-unit strides or broadcasts: |
| 1649 | /// |
| 1650 | /// * Transfer dimension is not the last memref dimension |
| 1651 | /// * Transfer dimension is a broadcast (i.e., scalar load + broadcast) |
| 1652 | /// * Memref has a layout map with non-unit stride on the last dimension |
| 1653 | /// |
| 1654 | /// This pattern generates IR as follows: |
| 1655 | /// |
| 1656 | /// 1. Generate a for loop iterating over each vector element. |
| 1657 | /// 2. Inside the loop, generate a InsertElementOp or ExtractElementOp, |
| 1658 | /// depending on OpTy. |
| 1659 | /// |
| 1660 | /// TODO: In some cases (no masking, etc.), LLVM::MatrixColumnMajorLoadOp |
| 1661 | /// can be generated instead of TransferOp1dConversion. Add such a pattern |
| 1662 | /// to ConvertVectorToLLVM. |
| 1663 | /// |
| 1664 | /// E.g.: |
| 1665 | /// ``` |
| 1666 | /// vector.transfer_write %vec, %A[%a, %b] |
| 1667 | /// {permutation_map = affine_map<(d0, d1) -> (d0)>, in_bounds = [true]} |
| 1668 | /// : vector<9xf32>, memref<?x?xf32> |
| 1669 | /// ``` |
| 1670 | /// Is rewritten to approximately the following pseudo-IR: |
| 1671 | /// ``` |
| 1672 | /// for i = 0 to 9 { |
| 1673 | /// %t = vector.extract %vec[i] : f32 from vector<9xf32> |
| 1674 | /// memref.store %t, %arg0[%a + i, %b] : memref<?x?xf32> |
| 1675 | /// } |
| 1676 | /// ``` |
| 1677 | template <typename OpTy> |
| 1678 | struct TransferOp1dConversion : public VectorToSCFPattern<OpTy> { |
| 1679 | using VectorToSCFPattern<OpTy>::VectorToSCFPattern; |
| 1680 | |
| 1681 | LogicalResult matchAndRewrite(OpTy xferOp, |
| 1682 | PatternRewriter &rewriter) const override { |
| 1683 | // TODO: support 0-d corner case. |
| 1684 | if (xferOp.getTransferRank() == 0) |
| 1685 | return failure(); |
| 1686 | if (failed(checkNotMasked(xferOp, rewriter))) |
| 1687 | return failure(); |
| 1688 | auto map = xferOp.getPermutationMap(); |
| 1689 | auto memRefType = dyn_cast<MemRefType>(xferOp.getShapedType()); |
| 1690 | |
| 1691 | if (!memRefType) |
| 1692 | return failure(); |
| 1693 | if (xferOp.getVectorType().getRank() != 1) |
| 1694 | return failure(); |
| 1695 | if (map.isMinorIdentity() && memRefType.isLastDimUnitStride()) |
| 1696 | return failure(); // Handled by ConvertVectorToLLVM |
| 1697 | |
| 1698 | // Loop bounds, step, state... |
| 1699 | Location loc = xferOp.getLoc(); |
| 1700 | auto vecType = xferOp.getVectorType(); |
| 1701 | auto lb = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1702 | Value ub = |
| 1703 | arith::ConstantIndexOp::create(rewriter, loc, vecType.getDimSize(0)); |
| 1704 | if (vecType.isScalable()) { |
| 1705 | Value vscale = |
| 1706 | vector::VectorScaleOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType()); |
| 1707 | ub = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: ub, rhs: vscale); |
| 1708 | } |
| 1709 | auto step = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1710 | auto loopState = Strategy1d<OpTy>::initialLoopState(rewriter, xferOp); |
| 1711 | |
| 1712 | // Generate for loop. |
| 1713 | rewriter.replaceOpWithNewOp<scf::ForOp>( |
| 1714 | xferOp, lb, ub, step, loopState ? ValueRange(loopState) : ValueRange(), |
| 1715 | [&](OpBuilder &b, Location loc, Value iv, ValueRange loopState) { |
| 1716 | Strategy1d<OpTy>::generateForLoopBody(b, loc, xferOp, iv, loopState); |
| 1717 | }); |
| 1718 | |
| 1719 | return success(); |
| 1720 | } |
| 1721 | }; |
| 1722 | |
| 1723 | } // namespace lowering_1_d |
| 1724 | } // namespace |
| 1725 | |
| 1726 | void mlir::populateVectorToSCFConversionPatterns( |
| 1727 | RewritePatternSet &patterns, const VectorTransferToSCFOptions &options) { |
| 1728 | if (options.unroll) { |
| 1729 | patterns.add<lowering_n_d_unrolled::UnrollTransferReadConversion, |
| 1730 | lowering_n_d_unrolled::UnrollTransferWriteConversion>( |
| 1731 | arg: patterns.getContext(), args: options); |
| 1732 | } else { |
| 1733 | patterns.add<lowering_n_d::PrepareTransferReadConversion, |
| 1734 | lowering_n_d::PrepareTransferWriteConversion, |
| 1735 | lowering_n_d::TransferOpConversion<TransferReadOp>, |
| 1736 | lowering_n_d::TransferOpConversion<TransferWriteOp>>( |
| 1737 | arg: patterns.getContext(), args: options); |
| 1738 | } |
| 1739 | if (options.lowerScalable) { |
| 1740 | patterns.add<lowering_n_d::ScalableTransposeTransferWriteConversion>( |
| 1741 | arg: patterns.getContext(), args: options); |
| 1742 | } |
| 1743 | if (options.targetRank == 1) { |
| 1744 | patterns.add<lowering_1_d::TransferOp1dConversion<TransferReadOp>, |
| 1745 | lowering_1_d::TransferOp1dConversion<TransferWriteOp>>( |
| 1746 | arg: patterns.getContext(), args: options); |
| 1747 | } |
| 1748 | patterns.add<lowering_n_d::DecomposePrintOpConversion>(arg: patterns.getContext(), |
| 1749 | args: options); |
| 1750 | } |
| 1751 | |
| 1752 | namespace { |
| 1753 | |
| 1754 | struct ConvertVectorToSCFPass |
| 1755 | : public impl::ConvertVectorToSCFBase<ConvertVectorToSCFPass> { |
| 1756 | ConvertVectorToSCFPass() = default; |
| 1757 | ConvertVectorToSCFPass(const VectorTransferToSCFOptions &options) { |
| 1758 | this->fullUnroll = options.unroll; |
| 1759 | this->targetRank = options.targetRank; |
| 1760 | this->lowerTensors = options.lowerTensors; |
| 1761 | this->lowerScalable = options.lowerScalable; |
| 1762 | } |
| 1763 | |
| 1764 | void runOnOperation() override { |
| 1765 | VectorTransferToSCFOptions options; |
| 1766 | options.unroll = fullUnroll; |
| 1767 | options.targetRank = targetRank; |
| 1768 | options.lowerTensors = lowerTensors; |
| 1769 | options.lowerScalable = lowerScalable; |
| 1770 | |
| 1771 | // Lower permutation maps first. |
| 1772 | RewritePatternSet lowerTransferPatterns(&getContext()); |
| 1773 | mlir::vector::populateVectorTransferPermutationMapLoweringPatterns( |
| 1774 | patterns&: lowerTransferPatterns); |
| 1775 | (void)applyPatternsGreedily(op: getOperation(), |
| 1776 | patterns: std::move(lowerTransferPatterns)); |
| 1777 | |
| 1778 | RewritePatternSet patterns(&getContext()); |
| 1779 | populateVectorToSCFConversionPatterns(patterns, options); |
| 1780 | (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)); |
| 1781 | } |
| 1782 | }; |
| 1783 | |
| 1784 | } // namespace |
| 1785 | |
| 1786 | std::unique_ptr<Pass> |
| 1787 | mlir::createConvertVectorToSCFPass(const VectorTransferToSCFOptions &options) { |
| 1788 | return std::make_unique<ConvertVectorToSCFPass>(args: options); |
| 1789 | } |
| 1790 | |