| 1 | //===- VectorToAMX.cpp - Convert vector to X86 dialect AMX ops --*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/VectorToAMX/VectorToAMX.h" |
| 10 | |
| 11 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" |
| 14 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 15 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 16 | #include "mlir/Dialect/Utils/StructuredOpsUtils.h" |
| 17 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 18 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 19 | #include "mlir/Dialect/X86/X86Dialect.h" |
| 20 | #include "mlir/IR/Builders.h" |
| 21 | #include "mlir/Pass/Pass.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | |
| 24 | #include "llvm/ADT/STLExtras.h" |
| 25 | #include "llvm/Support/DebugLog.h" |
| 26 | |
| 27 | #include <numeric> |
| 28 | |
| 29 | namespace mlir { |
| 30 | #define GEN_PASS_DEF_CONVERTVECTORTOAMX |
| 31 | #include "mlir/Conversion/Passes.h.inc" |
| 32 | } // namespace mlir |
| 33 | |
| 34 | using namespace mlir; |
| 35 | |
| 36 | #define DEBUG_TYPE "vector-to-amx" |
| 37 | |
| 38 | namespace { |
| 39 | |
| 40 | /// Return true if vector shape is compatible with AMX tiles. |
| 41 | /// The validation accounts for VNNI packing. |
| 42 | static bool verifyAmxShape(VectorType vec) { |
| 43 | // Check overall shape: |
| 44 | // - 2D for plain layout input or output |
| 45 | // - 3D for VNNI packed input |
| 46 | if (vec.getRank() != 2 && vec.getRank() != 3) |
| 47 | return false; |
| 48 | |
| 49 | ArrayRef<int64_t> shape = vec.getShape(); |
| 50 | int64_t rows = shape[0]; |
| 51 | int64_t cols = shape[1]; |
| 52 | unsigned elemBitWidth = vec.getElementType().getIntOrFloatBitWidth(); |
| 53 | |
| 54 | // 3D shape indicates VNNI packed layout. |
| 55 | if (vec.getRank() == 3) { |
| 56 | int64_t vnniFactor = 32 / elemBitWidth; |
| 57 | if (shape.back() != vnniFactor) { |
| 58 | LDBG() << "invalid VNNI packing factor" ; |
| 59 | return false; |
| 60 | } |
| 61 | cols *= vnniFactor; |
| 62 | } |
| 63 | |
| 64 | // AMX tile supports up to 16 rows of 64 bytes each. |
| 65 | constexpr unsigned maxRows = 16; |
| 66 | constexpr unsigned maxBitsPerRow = 64 * 8; |
| 67 | return rows <= maxRows && (cols * elemBitWidth) <= maxBitsPerRow; |
| 68 | } |
| 69 | |
| 70 | /// Check if contraction operands are in AMX-compatible packed VNNI layout. |
| 71 | static LogicalResult isAmxVnniLayout(PatternRewriter &rewriter, |
| 72 | vector::ContractionOp contractOp) { |
| 73 | VectorType accType = dyn_cast<VectorType>(Val: contractOp.getAcc().getType()); |
| 74 | if (!accType || accType.getRank() != 2) |
| 75 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Expects acc 2D vector" ); |
| 76 | |
| 77 | // Expect 3D inputs for VNNI packed data. |
| 78 | VectorType lhsType = contractOp.getLhs().getType(); |
| 79 | VectorType rhsType = contractOp.getRhs().getType(); |
| 80 | if (lhsType.getRank() != 3 || rhsType.getRank() != 3) |
| 81 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 82 | msg: "Expects lhs and rhs 3D vectors" ); |
| 83 | |
| 84 | // Check if shapes are compatible with AMX tile. |
| 85 | if (!verifyAmxShape(vec: lhsType) || !verifyAmxShape(vec: rhsType) || |
| 86 | !verifyAmxShape(vec: accType)) |
| 87 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid operand shape" ); |
| 88 | |
| 89 | // Validate affine maps. |
| 90 | // |
| 91 | // Iterators can be ordered arbitrarily. Indexing map positions are based on |
| 92 | // operands' target shapes. |
| 93 | // The matrix layouts must match the following: |
| 94 | // - matrix A - [M]x[K/vnniFactor]x[vnniFactor] |
| 95 | // - matrix B - [K/vnniFactor]x[N]x[vnniFactor] |
| 96 | // - matrix C - [M]x[N] |
| 97 | SmallVector<AffineMap, 4> indexingMaps = contractOp.getIndexingMapsArray(); |
| 98 | AffineMap mapA = indexingMaps[0]; |
| 99 | AffineMap mapB = indexingMaps[1]; |
| 100 | if (mapA.getNumInputs() != 4 || mapA.getNumResults() != 3 || |
| 101 | mapB.getNumResults() != 3) |
| 102 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 103 | msg: "Invalid input indexing maps" ); |
| 104 | FailureOr<linalg::ContractionDimensions> dims = |
| 105 | linalg::inferContractionDims(indexingMaps); |
| 106 | if (failed(Result: dims)) |
| 107 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 108 | msg: "Failed to infer contraction dims" ); |
| 109 | // Two reduction dimensions are expected: |
| 110 | // - one for the K dimension |
| 111 | // - one for the VNNI factor |
| 112 | if (dims->k.size() != 2) |
| 113 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 114 | msg: "Expected two reduction dims" ); |
| 115 | assert(dims->m.size() == 1 && dims->n.size() == 1 && |
| 116 | "Invalid parallel contraction dims" ); |
| 117 | |
| 118 | SmallVector<vector::IteratorType> iteratorTypes = |
| 119 | contractOp.getIteratorTypesArray(); |
| 120 | // Check VNNI dim maps - the innermost dim for A and B inputs. |
| 121 | auto vnniDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 2)); |
| 122 | auto vnniDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 2)); |
| 123 | if (!vnniDimA || !vnniDimB || vnniDimA != vnniDimB || |
| 124 | iteratorTypes[vnniDimA.getPosition()] != vector::IteratorType::reduction) |
| 125 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid VNNI dim map" ); |
| 126 | // Check K dim maps - non-transposed row-major layout. |
| 127 | auto redDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 1)); |
| 128 | auto redDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 0)); |
| 129 | if (!redDimA || !redDimB || redDimA != redDimB || |
| 130 | iteratorTypes[redDimA.getPosition()] != vector::IteratorType::reduction) |
| 131 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid K dim map" ); |
| 132 | // Check M and N dim maps - map to non-transposed output. |
| 133 | AffineMap mapC = indexingMaps[2]; |
| 134 | auto mDimC = dyn_cast<AffineDimExpr>(Val: mapC.getResult(idx: 0)); |
| 135 | auto nDimC = dyn_cast<AffineDimExpr>(Val: mapC.getResult(idx: 1)); |
| 136 | if (!mDimC || !nDimC) |
| 137 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid acc maps" ); |
| 138 | auto parallelDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 0)); |
| 139 | if (!parallelDimA || |
| 140 | iteratorTypes[parallelDimA.getPosition()] != |
| 141 | vector::IteratorType::parallel || |
| 142 | parallelDimA != mDimC) |
| 143 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid M dim map" ); |
| 144 | auto parallelDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 1)); |
| 145 | if (!parallelDimB || |
| 146 | iteratorTypes[parallelDimB.getPosition()] != |
| 147 | vector::IteratorType::parallel || |
| 148 | parallelDimB != nDimC) |
| 149 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid N dim map" ); |
| 150 | |
| 151 | return success(); |
| 152 | } |
| 153 | |
| 154 | /// Validate contraction operands for AMX lowering. |
| 155 | static LogicalResult validateOperands(PatternRewriter &rewriter, |
| 156 | vector::ContractionOp contractOp) { |
| 157 | VectorType accType = dyn_cast<VectorType>(Val: contractOp.getAcc().getType()); |
| 158 | if (!accType) |
| 159 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Expects vector acc" ); |
| 160 | |
| 161 | // Check if operand types are compatible with AMX compute ops. |
| 162 | bool validElemTypes = false; |
| 163 | Type lhsElemType = contractOp.getLhs().getType().getElementType(); |
| 164 | Type rhsElemType = contractOp.getRhs().getType().getElementType(); |
| 165 | Type accElemType = accType.getElementType(); |
| 166 | if (accElemType.isInteger(width: 32)) { |
| 167 | validElemTypes = lhsElemType.isInteger(width: 8) && rhsElemType.isInteger(width: 8); |
| 168 | } else if (accElemType.isF32()) { |
| 169 | validElemTypes = (lhsElemType.isF16() && rhsElemType.isF16()) || |
| 170 | (lhsElemType.isBF16() && rhsElemType.isBF16()); |
| 171 | } |
| 172 | if (!validElemTypes) |
| 173 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 174 | msg: "Invalid combination of operand types" ); |
| 175 | |
| 176 | if (failed(Result: isAmxVnniLayout(rewriter, contractOp))) |
| 177 | return failure(); |
| 178 | |
| 179 | return success(); |
| 180 | } |
| 181 | |
| 182 | /// Collapse the two innermost dimensions together. |
| 183 | static TypedValue<MemRefType> collapseLastDim(PatternRewriter &rewriter, |
| 184 | TypedValue<MemRefType> memref) { |
| 185 | int64_t rank = memref.getType().getRank(); |
| 186 | SmallVector<ReassociationIndices> reassocIndices; |
| 187 | for (auto i : llvm::seq<int64_t>(Begin: 0, End: rank - 2)) |
| 188 | reassocIndices.push_back(Elt: {i}); |
| 189 | reassocIndices.push_back(Elt: {rank - 2, rank - 1}); |
| 190 | return memref::CollapseShapeOp::create(builder&: rewriter, location: memref.getLoc(), src: memref, |
| 191 | reassociation: reassocIndices); |
| 192 | } |
| 193 | |
| 194 | /// Attempt to create an AMX tile load/store operation equivalent to the given |
| 195 | /// vector transfer `xfer` op. |
| 196 | /// This approach allows to skip longer route through registers and a temporary |
| 197 | /// buffer otherwise required to move data to/from an AMX tile. |
| 198 | static Operation * |
| 199 | loadStoreFromTransfer(PatternRewriter &rewriter, |
| 200 | VectorTransferOpInterface xferOp, bool isPacked, |
| 201 | TypedValue<x86::amx::TileType> tileToStore = nullptr) { |
| 202 | if (!xferOp || !isa<vector::TransferReadOp, vector::TransferWriteOp>(Val: xferOp)) |
| 203 | return nullptr; |
| 204 | if (xferOp.hasOutOfBoundsDim() || |
| 205 | !xferOp.getPermutationMap().isMinorIdentity()) |
| 206 | return nullptr; |
| 207 | |
| 208 | // Extra checks in case of a write op. |
| 209 | // Stores must not be packed. |
| 210 | if (isa<vector::TransferWriteOp>(Val: xferOp) && |
| 211 | (!tileToStore || isPacked || |
| 212 | tileToStore.getType().getShape() != xferOp.getVectorType().getShape())) |
| 213 | return nullptr; |
| 214 | |
| 215 | // Check for a memref source buffer. |
| 216 | // AMX data transfer requires at least 2D shape to correctly |
| 217 | // infer stride between rows. |
| 218 | Value base = xferOp.getBase(); |
| 219 | auto memTy = dyn_cast<MemRefType>(Val: base.getType()); |
| 220 | int64_t memRank = memTy.getRank(); |
| 221 | if (!memTy || memRank < 2) |
| 222 | return nullptr; |
| 223 | |
| 224 | // Check that the source buffer has enough contiguous elements to load whole |
| 225 | // AMX tile row. |
| 226 | // |
| 227 | // To ensure correctness, the validation is conservative and expects the |
| 228 | // buffer's innermost dimensions to be statically known, equal to or larger |
| 229 | // than the vector row length, and equal to the VNNI dimension if applicable. |
| 230 | // |
| 231 | // This check could be relaxed to accept more arbitrarily shaped buffers as |
| 232 | // long as there are enough contiguous elements to load a whole row. |
| 233 | if (!memTy.areTrailingDimsContiguous(n: isPacked ? 2 : 1)) |
| 234 | return nullptr; |
| 235 | VectorType vecTy = xferOp.getVectorType(); |
| 236 | ArrayRef<int64_t> vecShape = vecTy.getShape(); |
| 237 | ArrayRef<int64_t> memShape = memTy.getShape(); |
| 238 | if (memShape.back() == ShapedType::kDynamic || |
| 239 | memShape.back() < vecShape.back()) |
| 240 | return nullptr; |
| 241 | if (isPacked && |
| 242 | (memShape.back() != vecShape.back() || |
| 243 | memShape[memShape.size() - 2] == ShapedType::kDynamic || |
| 244 | memShape[memShape.size() - 2] < vecShape[vecShape.size() - 2])) |
| 245 | return nullptr; |
| 246 | |
| 247 | // Load values directly from the buffer to an AMX tile. |
| 248 | PatternRewriter::InsertionGuard g(rewriter); |
| 249 | rewriter.setInsertionPoint(xferOp); |
| 250 | Location loc = xferOp.getLoc(); |
| 251 | |
| 252 | // Create a subview of the source buffer based on the transfer op to resolve |
| 253 | // offsets. |
| 254 | SmallVector<OpFoldResult> strides(memRank, rewriter.getIndexAttr(value: 1)); |
| 255 | int64_t vecRank = vecTy.getRank(); |
| 256 | assert(memRank >= vecRank && |
| 257 | "Expects buffer to be the same or greater rank than vector" ); |
| 258 | SmallVector<int64_t> shape(memRank - vecRank, 1); |
| 259 | shape.append(in_start: vecShape.begin(), in_end: vecShape.end()); |
| 260 | TypedValue<MemRefType> src = |
| 261 | memref::SubViewOp::create( |
| 262 | builder&: rewriter, location: loc, source: base, offsets: getAsOpFoldResult(values: xferOp.getIndices()), |
| 263 | sizes: getAsOpFoldResult(arrayAttr: rewriter.getI64ArrayAttr(values: shape)), strides) |
| 264 | .getResult(); |
| 265 | |
| 266 | // Collapse the VNNI dimension in case of packing. |
| 267 | if (isPacked) |
| 268 | src = collapseLastDim(rewriter, memref: src); |
| 269 | int64_t rows = vecShape[0]; |
| 270 | int64_t cols = llvm::product_of(Range: vecShape.drop_front()); |
| 271 | auto tileType = x86::amx::TileType::get(shape: {rows, cols}, elementType: vecTy.getElementType()); |
| 272 | |
| 273 | Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0); |
| 274 | SmallVector<Value> tileIndicides(src.getType().getRank(), zeroIndex); |
| 275 | |
| 276 | Operation *amxTileOp = nullptr; |
| 277 | if (isa<vector::TransferReadOp>(Val: xferOp)) { |
| 278 | amxTileOp = x86::amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: src, |
| 279 | indices: tileIndicides); |
| 280 | } else if (isa<vector::TransferWriteOp>(Val: xferOp)) { |
| 281 | amxTileOp = x86::amx::TileStoreOp::create(builder&: rewriter, location: loc, base: src, indices: tileIndicides, |
| 282 | val: tileToStore); |
| 283 | } else { |
| 284 | llvm_unreachable("unsupported vector transfer op" ); |
| 285 | } |
| 286 | |
| 287 | return amxTileOp; |
| 288 | } |
| 289 | |
| 290 | /// Attempt to create an AMX tile load operation equivalent to the given |
| 291 | /// vector transfer `readOp`. |
| 292 | /// Returns loaded AMX tile if successful. |
| 293 | static FailureOr<TypedValue<x86::amx::TileType>> |
| 294 | loadFromTransfer(PatternRewriter &rewriter, vector::TransferReadOp readOp, |
| 295 | bool isPacked) { |
| 296 | x86::amx::TileLoadOp loadOp = dyn_cast_if_present<x86::amx::TileLoadOp>( |
| 297 | Val: loadStoreFromTransfer(rewriter, xferOp: readOp, isPacked)); |
| 298 | if (!loadOp) |
| 299 | return failure(); |
| 300 | return loadOp.getRes(); |
| 301 | } |
| 302 | |
| 303 | /// Attempt to create an AMX tile store operation equivalent to the given |
| 304 | /// vector transfer `writeOp`. |
| 305 | static LogicalResult |
| 306 | storeFromTransfer(PatternRewriter &rewriter, vector::TransferWriteOp writeOp, |
| 307 | TypedValue<x86::amx::TileType> tileToStore) { |
| 308 | return success(IsSuccess: loadStoreFromTransfer(rewriter, xferOp: writeOp, /*isPacked=*/false, |
| 309 | tileToStore)); |
| 310 | } |
| 311 | |
| 312 | /// Load vector values to an AMX tile. |
| 313 | static TypedValue<x86::amx::TileType> loadTile(PatternRewriter &rewriter, |
| 314 | TypedValue<VectorType> vec) { |
| 315 | Location loc = vec.getLoc(); |
| 316 | |
| 317 | VectorType vecTy = vec.getType(); |
| 318 | bool isPacked = vecTy.getRank() == 3; |
| 319 | |
| 320 | // Try to load tile directly from vector producer's buffer. |
| 321 | auto readOp = vec.getDefiningOp<vector::TransferReadOp>(); |
| 322 | FailureOr<TypedValue<x86::amx::TileType>> tile = |
| 323 | loadFromTransfer(rewriter, readOp, isPacked); |
| 324 | if (succeeded(Result: tile)) |
| 325 | return *tile; |
| 326 | |
| 327 | // Transfer the vector to a tile through an intermediate buffer. |
| 328 | Value buf = memref::AllocaOp::create( |
| 329 | builder&: rewriter, location: loc, memrefType: MemRefType::get(shape: vecTy.getShape(), elementType: vecTy.getElementType())); |
| 330 | Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0); |
| 331 | SmallVector<Value> indices(vecTy.getRank(), zeroIndex); |
| 332 | vector::TransferWriteOp::create(builder&: rewriter, location: loc, vector: vec, dest: buf, indices); |
| 333 | |
| 334 | // Collapse the VNNI dimension in case of packing. |
| 335 | if (isPacked) |
| 336 | buf = collapseLastDim(rewriter, memref: cast<TypedValue<MemRefType>>(Val&: buf)); |
| 337 | |
| 338 | ArrayRef<int64_t> shape = vecTy.getShape(); |
| 339 | int64_t rows = shape[0]; |
| 340 | int64_t cols = llvm::product_of(Range: shape.drop_front()); |
| 341 | auto tileType = x86::amx::TileType::get(shape: {rows, cols}, elementType: vecTy.getElementType()); |
| 342 | |
| 343 | return x86::amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: buf, |
| 344 | indices: {zeroIndex, zeroIndex}); |
| 345 | } |
| 346 | |
| 347 | /// Store an AMX tile in a vector. |
| 348 | static TypedValue<VectorType> storeTile(PatternRewriter &rewriter, |
| 349 | TypedValue<x86::amx::TileType> tile) { |
| 350 | Location loc = tile.getLoc(); |
| 351 | |
| 352 | // Transfer the tile to a vector through an intermediate buffer. |
| 353 | x86::amx::TileType tileTy = tile.getType(); |
| 354 | Value buf = memref::AllocaOp::create( |
| 355 | builder&: rewriter, location: loc, |
| 356 | memrefType: MemRefType::get(shape: tileTy.getShape(), elementType: tileTy.getElementType())); |
| 357 | Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0); |
| 358 | Repeated<Value> indices(2, zeroIndex); |
| 359 | x86::amx::TileStoreOp::create(builder&: rewriter, location: loc, base: buf, indices, val: tile); |
| 360 | |
| 361 | auto vecTy = VectorType::get(shape: tileTy.getShape(), elementType: tileTy.getElementType()); |
| 362 | return vector::TransferReadOp::create(builder&: rewriter, location: loc, vectorType: vecTy, source: buf, indices, padding: {}); |
| 363 | } |
| 364 | |
| 365 | struct ContractionToAMX : public vector::AddContractionOpRewritePattern { |
| 366 | using Base::Base; |
| 367 | |
| 368 | LogicalResult |
| 369 | matchAndRewriteAddContractOp(vector::ContractionOp contractOp, |
| 370 | PatternRewriter &rewriter) const override { |
| 371 | Location loc = contractOp.getLoc(); |
| 372 | |
| 373 | if (failed(Result: validateOperands(rewriter, contractOp))) |
| 374 | return failure(); |
| 375 | |
| 376 | TypedValue<x86::amx::TileType> lhsTile = |
| 377 | loadTile(rewriter, vec: contractOp.getLhs()); |
| 378 | TypedValue<x86::amx::TileType> rhsTile = |
| 379 | loadTile(rewriter, vec: contractOp.getRhs()); |
| 380 | auto acc = dyn_cast<TypedValue<VectorType>>(Val: contractOp.getAcc()); |
| 381 | assert(acc && "Invalid accumulator type" ); |
| 382 | TypedValue<x86::amx::TileType> accTile = loadTile(rewriter, vec: acc); |
| 383 | |
| 384 | TypedValue<x86::amx::TileType> tileMul; |
| 385 | if (acc.getType().getElementType().isFloat()) { |
| 386 | tileMul = x86::amx::TileMulFOp::create(builder&: rewriter, location: loc, res: accTile.getType(), |
| 387 | lhs: lhsTile, rhs: rhsTile, acc: accTile); |
| 388 | } else { |
| 389 | tileMul = x86::amx::TileMulIOp::create(builder&: rewriter, location: loc, res: accTile.getType(), |
| 390 | lhs: lhsTile, rhs: rhsTile, acc: accTile); |
| 391 | } |
| 392 | |
| 393 | // If the contraction result is only written back to memory, try to replace |
| 394 | // the vector op with an AMX store directly. |
| 395 | Value res = contractOp.getResult(); |
| 396 | if (res.hasOneUse()) { |
| 397 | auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: *res.getUsers().begin()); |
| 398 | LogicalResult storeRes = storeFromTransfer(rewriter, writeOp, tileToStore: tileMul); |
| 399 | if (succeeded(Result: storeRes)) { |
| 400 | rewriter.eraseOp(op: writeOp); |
| 401 | rewriter.eraseOp(op: contractOp); |
| 402 | return success(); |
| 403 | } |
| 404 | } |
| 405 | |
| 406 | // Load the result back into a vector. |
| 407 | Value newResult = storeTile(rewriter, tile: tileMul); |
| 408 | rewriter.replaceOp(op: contractOp, newValues: newResult); |
| 409 | |
| 410 | return success(); |
| 411 | } |
| 412 | }; |
| 413 | |
| 414 | struct ConvertVectorToAMXPass |
| 415 | : public impl::ConvertVectorToAMXBase<ConvertVectorToAMXPass> { |
| 416 | void runOnOperation() override { |
| 417 | MLIRContext &ctx = getContext(); |
| 418 | RewritePatternSet patterns(&ctx); |
| 419 | populateVectorToAMXConversionPatterns(patterns); |
| 420 | if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) |
| 421 | return signalPassFailure(); |
| 422 | } |
| 423 | }; |
| 424 | |
| 425 | } // namespace |
| 426 | |
| 427 | void mlir::populateVectorToAMXConversionPatterns(RewritePatternSet &patterns) { |
| 428 | patterns.add<ContractionToAMX>(arg: patterns.getContext()); |
| 429 | } |
| 430 | |