1//===- VectorToAMX.cpp - Convert vector to X86 dialect AMX ops --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/VectorToAMX/VectorToAMX.h"
10
11#include "mlir/Dialect/Affine/IR/AffineOps.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
14#include "mlir/Dialect/MemRef/IR/MemRef.h"
15#include "mlir/Dialect/SCF/IR/SCF.h"
16#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
17#include "mlir/Dialect/Vector/IR/VectorOps.h"
18#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
19#include "mlir/Dialect/X86/X86Dialect.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/Support/DebugLog.h"
26
27#include <numeric>
28
29namespace mlir {
30#define GEN_PASS_DEF_CONVERTVECTORTOAMX
31#include "mlir/Conversion/Passes.h.inc"
32} // namespace mlir
33
34using namespace mlir;
35
36#define DEBUG_TYPE "vector-to-amx"
37
38namespace {
39
40/// Return true if vector shape is compatible with AMX tiles.
41/// The validation accounts for VNNI packing.
42static bool verifyAmxShape(VectorType vec) {
43 // Check overall shape:
44 // - 2D for plain layout input or output
45 // - 3D for VNNI packed input
46 if (vec.getRank() != 2 && vec.getRank() != 3)
47 return false;
48
49 ArrayRef<int64_t> shape = vec.getShape();
50 int64_t rows = shape[0];
51 int64_t cols = shape[1];
52 unsigned elemBitWidth = vec.getElementType().getIntOrFloatBitWidth();
53
54 // 3D shape indicates VNNI packed layout.
55 if (vec.getRank() == 3) {
56 int64_t vnniFactor = 32 / elemBitWidth;
57 if (shape.back() != vnniFactor) {
58 LDBG() << "invalid VNNI packing factor";
59 return false;
60 }
61 cols *= vnniFactor;
62 }
63
64 // AMX tile supports up to 16 rows of 64 bytes each.
65 constexpr unsigned maxRows = 16;
66 constexpr unsigned maxBitsPerRow = 64 * 8;
67 return rows <= maxRows && (cols * elemBitWidth) <= maxBitsPerRow;
68}
69
70/// Check if contraction operands are in AMX-compatible packed VNNI layout.
71static LogicalResult isAmxVnniLayout(PatternRewriter &rewriter,
72 vector::ContractionOp contractOp) {
73 VectorType accType = dyn_cast<VectorType>(Val: contractOp.getAcc().getType());
74 if (!accType || accType.getRank() != 2)
75 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Expects acc 2D vector");
76
77 // Expect 3D inputs for VNNI packed data.
78 VectorType lhsType = contractOp.getLhs().getType();
79 VectorType rhsType = contractOp.getRhs().getType();
80 if (lhsType.getRank() != 3 || rhsType.getRank() != 3)
81 return rewriter.notifyMatchFailure(arg&: contractOp,
82 msg: "Expects lhs and rhs 3D vectors");
83
84 // Check if shapes are compatible with AMX tile.
85 if (!verifyAmxShape(vec: lhsType) || !verifyAmxShape(vec: rhsType) ||
86 !verifyAmxShape(vec: accType))
87 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid operand shape");
88
89 // Validate affine maps.
90 //
91 // Iterators can be ordered arbitrarily. Indexing map positions are based on
92 // operands' target shapes.
93 // The matrix layouts must match the following:
94 // - matrix A - [M]x[K/vnniFactor]x[vnniFactor]
95 // - matrix B - [K/vnniFactor]x[N]x[vnniFactor]
96 // - matrix C - [M]x[N]
97 SmallVector<AffineMap, 4> indexingMaps = contractOp.getIndexingMapsArray();
98 AffineMap mapA = indexingMaps[0];
99 AffineMap mapB = indexingMaps[1];
100 if (mapA.getNumInputs() != 4 || mapA.getNumResults() != 3 ||
101 mapB.getNumResults() != 3)
102 return rewriter.notifyMatchFailure(arg&: contractOp,
103 msg: "Invalid input indexing maps");
104 FailureOr<linalg::ContractionDimensions> dims =
105 linalg::inferContractionDims(indexingMaps);
106 if (failed(Result: dims))
107 return rewriter.notifyMatchFailure(arg&: contractOp,
108 msg: "Failed to infer contraction dims");
109 // Two reduction dimensions are expected:
110 // - one for the K dimension
111 // - one for the VNNI factor
112 if (dims->k.size() != 2)
113 return rewriter.notifyMatchFailure(arg&: contractOp,
114 msg: "Expected two reduction dims");
115 assert(dims->m.size() == 1 && dims->n.size() == 1 &&
116 "Invalid parallel contraction dims");
117
118 SmallVector<vector::IteratorType> iteratorTypes =
119 contractOp.getIteratorTypesArray();
120 // Check VNNI dim maps - the innermost dim for A and B inputs.
121 auto vnniDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 2));
122 auto vnniDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 2));
123 if (!vnniDimA || !vnniDimB || vnniDimA != vnniDimB ||
124 iteratorTypes[vnniDimA.getPosition()] != vector::IteratorType::reduction)
125 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid VNNI dim map");
126 // Check K dim maps - non-transposed row-major layout.
127 auto redDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 1));
128 auto redDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 0));
129 if (!redDimA || !redDimB || redDimA != redDimB ||
130 iteratorTypes[redDimA.getPosition()] != vector::IteratorType::reduction)
131 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid K dim map");
132 // Check M and N dim maps - map to non-transposed output.
133 AffineMap mapC = indexingMaps[2];
134 auto mDimC = dyn_cast<AffineDimExpr>(Val: mapC.getResult(idx: 0));
135 auto nDimC = dyn_cast<AffineDimExpr>(Val: mapC.getResult(idx: 1));
136 if (!mDimC || !nDimC)
137 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid acc maps");
138 auto parallelDimA = dyn_cast<AffineDimExpr>(Val: mapA.getResult(idx: 0));
139 if (!parallelDimA ||
140 iteratorTypes[parallelDimA.getPosition()] !=
141 vector::IteratorType::parallel ||
142 parallelDimA != mDimC)
143 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid M dim map");
144 auto parallelDimB = dyn_cast<AffineDimExpr>(Val: mapB.getResult(idx: 1));
145 if (!parallelDimB ||
146 iteratorTypes[parallelDimB.getPosition()] !=
147 vector::IteratorType::parallel ||
148 parallelDimB != nDimC)
149 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid N dim map");
150
151 return success();
152}
153
154/// Validate contraction operands for AMX lowering.
155static LogicalResult validateOperands(PatternRewriter &rewriter,
156 vector::ContractionOp contractOp) {
157 VectorType accType = dyn_cast<VectorType>(Val: contractOp.getAcc().getType());
158 if (!accType)
159 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Expects vector acc");
160
161 // Check if operand types are compatible with AMX compute ops.
162 bool validElemTypes = false;
163 Type lhsElemType = contractOp.getLhs().getType().getElementType();
164 Type rhsElemType = contractOp.getRhs().getType().getElementType();
165 Type accElemType = accType.getElementType();
166 if (accElemType.isInteger(width: 32)) {
167 validElemTypes = lhsElemType.isInteger(width: 8) && rhsElemType.isInteger(width: 8);
168 } else if (accElemType.isF32()) {
169 validElemTypes = (lhsElemType.isF16() && rhsElemType.isF16()) ||
170 (lhsElemType.isBF16() && rhsElemType.isBF16());
171 }
172 if (!validElemTypes)
173 return rewriter.notifyMatchFailure(arg&: contractOp,
174 msg: "Invalid combination of operand types");
175
176 if (failed(Result: isAmxVnniLayout(rewriter, contractOp)))
177 return failure();
178
179 return success();
180}
181
182/// Collapse the two innermost dimensions together.
183static TypedValue<MemRefType> collapseLastDim(PatternRewriter &rewriter,
184 TypedValue<MemRefType> memref) {
185 int64_t rank = memref.getType().getRank();
186 SmallVector<ReassociationIndices> reassocIndices;
187 for (auto i : llvm::seq<int64_t>(Begin: 0, End: rank - 2))
188 reassocIndices.push_back(Elt: {i});
189 reassocIndices.push_back(Elt: {rank - 2, rank - 1});
190 return memref::CollapseShapeOp::create(builder&: rewriter, location: memref.getLoc(), src: memref,
191 reassociation: reassocIndices);
192}
193
194/// Attempt to create an AMX tile load/store operation equivalent to the given
195/// vector transfer `xfer` op.
196/// This approach allows to skip longer route through registers and a temporary
197/// buffer otherwise required to move data to/from an AMX tile.
198static Operation *
199loadStoreFromTransfer(PatternRewriter &rewriter,
200 VectorTransferOpInterface xferOp, bool isPacked,
201 TypedValue<x86::amx::TileType> tileToStore = nullptr) {
202 if (!xferOp || !isa<vector::TransferReadOp, vector::TransferWriteOp>(Val: xferOp))
203 return nullptr;
204 if (xferOp.hasOutOfBoundsDim() ||
205 !xferOp.getPermutationMap().isMinorIdentity())
206 return nullptr;
207
208 // Extra checks in case of a write op.
209 // Stores must not be packed.
210 if (isa<vector::TransferWriteOp>(Val: xferOp) &&
211 (!tileToStore || isPacked ||
212 tileToStore.getType().getShape() != xferOp.getVectorType().getShape()))
213 return nullptr;
214
215 // Check for a memref source buffer.
216 // AMX data transfer requires at least 2D shape to correctly
217 // infer stride between rows.
218 Value base = xferOp.getBase();
219 auto memTy = dyn_cast<MemRefType>(Val: base.getType());
220 int64_t memRank = memTy.getRank();
221 if (!memTy || memRank < 2)
222 return nullptr;
223
224 // Check that the source buffer has enough contiguous elements to load whole
225 // AMX tile row.
226 //
227 // To ensure correctness, the validation is conservative and expects the
228 // buffer's innermost dimensions to be statically known, equal to or larger
229 // than the vector row length, and equal to the VNNI dimension if applicable.
230 //
231 // This check could be relaxed to accept more arbitrarily shaped buffers as
232 // long as there are enough contiguous elements to load a whole row.
233 if (!memTy.areTrailingDimsContiguous(n: isPacked ? 2 : 1))
234 return nullptr;
235 VectorType vecTy = xferOp.getVectorType();
236 ArrayRef<int64_t> vecShape = vecTy.getShape();
237 ArrayRef<int64_t> memShape = memTy.getShape();
238 if (memShape.back() == ShapedType::kDynamic ||
239 memShape.back() < vecShape.back())
240 return nullptr;
241 if (isPacked &&
242 (memShape.back() != vecShape.back() ||
243 memShape[memShape.size() - 2] == ShapedType::kDynamic ||
244 memShape[memShape.size() - 2] < vecShape[vecShape.size() - 2]))
245 return nullptr;
246
247 // Load values directly from the buffer to an AMX tile.
248 PatternRewriter::InsertionGuard g(rewriter);
249 rewriter.setInsertionPoint(xferOp);
250 Location loc = xferOp.getLoc();
251
252 // Create a subview of the source buffer based on the transfer op to resolve
253 // offsets.
254 SmallVector<OpFoldResult> strides(memRank, rewriter.getIndexAttr(value: 1));
255 int64_t vecRank = vecTy.getRank();
256 assert(memRank >= vecRank &&
257 "Expects buffer to be the same or greater rank than vector");
258 SmallVector<int64_t> shape(memRank - vecRank, 1);
259 shape.append(in_start: vecShape.begin(), in_end: vecShape.end());
260 TypedValue<MemRefType> src =
261 memref::SubViewOp::create(
262 builder&: rewriter, location: loc, source: base, offsets: getAsOpFoldResult(values: xferOp.getIndices()),
263 sizes: getAsOpFoldResult(arrayAttr: rewriter.getI64ArrayAttr(values: shape)), strides)
264 .getResult();
265
266 // Collapse the VNNI dimension in case of packing.
267 if (isPacked)
268 src = collapseLastDim(rewriter, memref: src);
269 int64_t rows = vecShape[0];
270 int64_t cols = llvm::product_of(Range: vecShape.drop_front());
271 auto tileType = x86::amx::TileType::get(shape: {rows, cols}, elementType: vecTy.getElementType());
272
273 Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0);
274 SmallVector<Value> tileIndicides(src.getType().getRank(), zeroIndex);
275
276 Operation *amxTileOp = nullptr;
277 if (isa<vector::TransferReadOp>(Val: xferOp)) {
278 amxTileOp = x86::amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: src,
279 indices: tileIndicides);
280 } else if (isa<vector::TransferWriteOp>(Val: xferOp)) {
281 amxTileOp = x86::amx::TileStoreOp::create(builder&: rewriter, location: loc, base: src, indices: tileIndicides,
282 val: tileToStore);
283 } else {
284 llvm_unreachable("unsupported vector transfer op");
285 }
286
287 return amxTileOp;
288}
289
290/// Attempt to create an AMX tile load operation equivalent to the given
291/// vector transfer `readOp`.
292/// Returns loaded AMX tile if successful.
293static FailureOr<TypedValue<x86::amx::TileType>>
294loadFromTransfer(PatternRewriter &rewriter, vector::TransferReadOp readOp,
295 bool isPacked) {
296 x86::amx::TileLoadOp loadOp = dyn_cast_if_present<x86::amx::TileLoadOp>(
297 Val: loadStoreFromTransfer(rewriter, xferOp: readOp, isPacked));
298 if (!loadOp)
299 return failure();
300 return loadOp.getRes();
301}
302
303/// Attempt to create an AMX tile store operation equivalent to the given
304/// vector transfer `writeOp`.
305static LogicalResult
306storeFromTransfer(PatternRewriter &rewriter, vector::TransferWriteOp writeOp,
307 TypedValue<x86::amx::TileType> tileToStore) {
308 return success(IsSuccess: loadStoreFromTransfer(rewriter, xferOp: writeOp, /*isPacked=*/false,
309 tileToStore));
310}
311
312/// Load vector values to an AMX tile.
313static TypedValue<x86::amx::TileType> loadTile(PatternRewriter &rewriter,
314 TypedValue<VectorType> vec) {
315 Location loc = vec.getLoc();
316
317 VectorType vecTy = vec.getType();
318 bool isPacked = vecTy.getRank() == 3;
319
320 // Try to load tile directly from vector producer's buffer.
321 auto readOp = vec.getDefiningOp<vector::TransferReadOp>();
322 FailureOr<TypedValue<x86::amx::TileType>> tile =
323 loadFromTransfer(rewriter, readOp, isPacked);
324 if (succeeded(Result: tile))
325 return *tile;
326
327 // Transfer the vector to a tile through an intermediate buffer.
328 Value buf = memref::AllocaOp::create(
329 builder&: rewriter, location: loc, memrefType: MemRefType::get(shape: vecTy.getShape(), elementType: vecTy.getElementType()));
330 Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0);
331 SmallVector<Value> indices(vecTy.getRank(), zeroIndex);
332 vector::TransferWriteOp::create(builder&: rewriter, location: loc, vector: vec, dest: buf, indices);
333
334 // Collapse the VNNI dimension in case of packing.
335 if (isPacked)
336 buf = collapseLastDim(rewriter, memref: cast<TypedValue<MemRefType>>(Val&: buf));
337
338 ArrayRef<int64_t> shape = vecTy.getShape();
339 int64_t rows = shape[0];
340 int64_t cols = llvm::product_of(Range: shape.drop_front());
341 auto tileType = x86::amx::TileType::get(shape: {rows, cols}, elementType: vecTy.getElementType());
342
343 return x86::amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: buf,
344 indices: {zeroIndex, zeroIndex});
345}
346
347/// Store an AMX tile in a vector.
348static TypedValue<VectorType> storeTile(PatternRewriter &rewriter,
349 TypedValue<x86::amx::TileType> tile) {
350 Location loc = tile.getLoc();
351
352 // Transfer the tile to a vector through an intermediate buffer.
353 x86::amx::TileType tileTy = tile.getType();
354 Value buf = memref::AllocaOp::create(
355 builder&: rewriter, location: loc,
356 memrefType: MemRefType::get(shape: tileTy.getShape(), elementType: tileTy.getElementType()));
357 Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(location: loc, args: 0);
358 Repeated<Value> indices(2, zeroIndex);
359 x86::amx::TileStoreOp::create(builder&: rewriter, location: loc, base: buf, indices, val: tile);
360
361 auto vecTy = VectorType::get(shape: tileTy.getShape(), elementType: tileTy.getElementType());
362 return vector::TransferReadOp::create(builder&: rewriter, location: loc, vectorType: vecTy, source: buf, indices, padding: {});
363}
364
365struct ContractionToAMX : public vector::AddContractionOpRewritePattern {
366 using Base::Base;
367
368 LogicalResult
369 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
370 PatternRewriter &rewriter) const override {
371 Location loc = contractOp.getLoc();
372
373 if (failed(Result: validateOperands(rewriter, contractOp)))
374 return failure();
375
376 TypedValue<x86::amx::TileType> lhsTile =
377 loadTile(rewriter, vec: contractOp.getLhs());
378 TypedValue<x86::amx::TileType> rhsTile =
379 loadTile(rewriter, vec: contractOp.getRhs());
380 auto acc = dyn_cast<TypedValue<VectorType>>(Val: contractOp.getAcc());
381 assert(acc && "Invalid accumulator type");
382 TypedValue<x86::amx::TileType> accTile = loadTile(rewriter, vec: acc);
383
384 TypedValue<x86::amx::TileType> tileMul;
385 if (acc.getType().getElementType().isFloat()) {
386 tileMul = x86::amx::TileMulFOp::create(builder&: rewriter, location: loc, res: accTile.getType(),
387 lhs: lhsTile, rhs: rhsTile, acc: accTile);
388 } else {
389 tileMul = x86::amx::TileMulIOp::create(builder&: rewriter, location: loc, res: accTile.getType(),
390 lhs: lhsTile, rhs: rhsTile, acc: accTile);
391 }
392
393 // If the contraction result is only written back to memory, try to replace
394 // the vector op with an AMX store directly.
395 Value res = contractOp.getResult();
396 if (res.hasOneUse()) {
397 auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: *res.getUsers().begin());
398 LogicalResult storeRes = storeFromTransfer(rewriter, writeOp, tileToStore: tileMul);
399 if (succeeded(Result: storeRes)) {
400 rewriter.eraseOp(op: writeOp);
401 rewriter.eraseOp(op: contractOp);
402 return success();
403 }
404 }
405
406 // Load the result back into a vector.
407 Value newResult = storeTile(rewriter, tile: tileMul);
408 rewriter.replaceOp(op: contractOp, newValues: newResult);
409
410 return success();
411 }
412};
413
414struct ConvertVectorToAMXPass
415 : public impl::ConvertVectorToAMXBase<ConvertVectorToAMXPass> {
416 void runOnOperation() override {
417 MLIRContext &ctx = getContext();
418 RewritePatternSet patterns(&ctx);
419 populateVectorToAMXConversionPatterns(patterns);
420 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
421 return signalPassFailure();
422 }
423};
424
425} // namespace
426
427void mlir::populateVectorToAMXConversionPatterns(RewritePatternSet &patterns) {
428 patterns.add<ContractionToAMX>(arg: patterns.getContext());
429}
430