1//===- X86Dialect.cpp - MLIR X86 ops implementation -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86 dialect and its operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/X86/X86Dialect.h"
14#include "mlir/Conversion/LLVMCommon/Pattern.h"
15#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
18#include "mlir/IR/Builders.h"
19#include "mlir/IR/OpImplementation.h"
20#include "mlir/IR/TypeUtilities.h"
21
22using namespace mlir;
23
24#include "mlir/Dialect/X86/X86Interfaces.cpp.inc"
25
26#include "mlir/Dialect/X86/X86Dialect.cpp.inc"
27
28void x86::X86Dialect::initialize() {
29 addTypes<
30#define GET_TYPEDEF_LIST
31#include "mlir/Dialect/X86/X86Types.cpp.inc"
32 >();
33
34 addOperations<
35#define GET_OP_LIST
36#include "mlir/Dialect/X86/X86.cpp.inc"
37 >();
38}
39
40static Value getMemrefBuffPtr(Location loc, MemRefType type, Value buffer,
41 const LLVMTypeConverter &typeConverter,
42 RewriterBase &rewriter) {
43 MemRefDescriptor memRefDescriptor(buffer);
44 return memRefDescriptor.bufferPtr(builder&: rewriter, loc, converter: typeConverter, type);
45}
46
47LogicalResult x86::avx512::MaskCompressOp::verify() {
48 if (getSrc() && getConstantSrc())
49 return emitError(message: "cannot use both src and constant_src");
50
51 if (getSrc() && (getSrc().getType() != getDst().getType()))
52 return emitError(message: "failed to verify that src and dst have same type");
53
54 if (getConstantSrc() && (getConstantSrc()->getType() != getDst().getType()))
55 return emitError(
56 message: "failed to verify that constant_src and dst have same type");
57
58 return success();
59}
60
61SmallVector<Value> x86::avx512::MaskCompressOp::getIntrinsicOperands(
62 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
63 RewriterBase &rewriter) {
64 auto loc = getLoc();
65 Adaptor adaptor(operands, *this);
66
67 auto opType = adaptor.getA().getType();
68 Value src;
69 if (adaptor.getSrc()) {
70 src = adaptor.getSrc();
71 } else if (adaptor.getConstantSrc()) {
72 src = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: opType,
73 value: adaptor.getConstantSrcAttr());
74 } else {
75 auto zeroAttr = rewriter.getZeroAttr(type: opType);
76 src = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: opType, value: zeroAttr);
77 }
78
79 return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()};
80}
81
82SmallVector<Value>
83x86::avx::DotOp::getIntrinsicOperands(ArrayRef<Value> operands,
84 const LLVMTypeConverter &typeConverter,
85 RewriterBase &rewriter) {
86 SmallVector<Value> intrinsicOperands(operands);
87 // Dot product of all elements, broadcasted to all elements.
88 Value scale =
89 LLVM::ConstantOp::create(builder&: rewriter, location: getLoc(), type: rewriter.getI8Type(), value: 0xff);
90 intrinsicOperands.push_back(Elt: scale);
91
92 return intrinsicOperands;
93}
94
95SmallVector<Value> x86::avx::BcstToPackedF32Op::getIntrinsicOperands(
96 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
97 RewriterBase &rewriter) {
98 Adaptor adaptor(operands, *this);
99 return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(),
100 typeConverter, rewriter)};
101}
102
103SmallVector<Value> x86::avx::CvtPackedEvenIndexedToF32Op::getIntrinsicOperands(
104 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
105 RewriterBase &rewriter) {
106 Adaptor adaptor(operands, *this);
107 return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(),
108 typeConverter, rewriter)};
109}
110
111SmallVector<Value> x86::avx::CvtPackedOddIndexedToF32Op::getIntrinsicOperands(
112 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
113 RewriterBase &rewriter) {
114 Adaptor adaptor(operands, *this);
115 return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(),
116 typeConverter, rewriter)};
117}
118
119/// Verify that AMX supports the implied tile shape.
120static LogicalResult verifyTileSize(Operation *op, x86::amx::TileType tp) {
121 const unsigned kMaxRows = 16;
122 const unsigned kBitsPerRow = 64 * 8;
123 unsigned col = tp.getDimSize(idx: 1) * tp.getElementType().getIntOrFloatBitWidth();
124 if (tp.getDimSize(idx: 0) > kMaxRows)
125 return op->emitOpError(message: "bad row height: ") << tp.getDimSize(idx: 0);
126 if (col > kBitsPerRow || col & 0x1f)
127 return op->emitOpError(message: "bad column width: ") << (col >> 3);
128 return success();
129}
130
131/// Verify that AMX supports the multiplication.
132static LogicalResult verifyMultShape(Operation *op, x86::amx::TileType atp,
133 x86::amx::TileType btp,
134 x86::amx::TileType ctp, unsigned scale) {
135 unsigned am = atp.getDimSize(idx: 0), ak = atp.getDimSize(idx: 1) >> scale;
136 unsigned bk = btp.getDimSize(idx: 0), bn = btp.getDimSize(idx: 1) >> scale;
137 unsigned cm = ctp.getDimSize(idx: 0), cn = ctp.getDimSize(idx: 1);
138 if (cm != am || cn != bn || ak != bk)
139 return op->emitOpError(message: "bad mult shape: ")
140 << cm << " x " << cn << " x " << ak;
141 return success();
142}
143
144/// Maps the 2-dim vector shape to the two 16-bit tile sizes. The first
145/// dimension directly translates into the number of rows of the tiles.
146/// The second dimensions needs to be scaled by the number of bytes.
147static SmallVector<Value> getTileSizes(Location loc, x86::amx::TileType tType,
148 RewriterBase &rewriter) {
149 Type llvmInt16Type = rewriter.getIntegerType(width: 16);
150 unsigned width = tType.getElementType().getIntOrFloatBitWidth();
151 assert(llvm::isPowerOf2_64(width) && width >= 8);
152 unsigned bytes = width >> 3;
153 auto mattr = rewriter.getI16IntegerAttr(value: tType.getDimSize(idx: 0));
154 auto nattr = rewriter.getI16IntegerAttr(value: tType.getDimSize(idx: 1) * bytes);
155 return SmallVector<Value>{
156 LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt16Type, value: mattr),
157 LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt16Type, value: nattr)};
158}
159
160/// Returns stride expressed in number of bytes for the given `elementStride`
161/// stride encoded in number of elements of the type `mType`.
162static Value computeStrideInBytes(Location loc, MemRefType mType,
163 Value elementStride, RewriterBase &rewriter) {
164 Type llvmInt64Type = rewriter.getIntegerType(width: 64);
165 unsigned bytes = mType.getElementType().getIntOrFloatBitWidth() / 8;
166 auto attr = rewriter.getI64IntegerAttr(value: bytes);
167 Value scale = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, value: attr);
168 return LLVM::MulOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, lhs: scale, rhs: elementStride)
169 .getResult();
170}
171
172/// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer
173/// shape may "envelop" the actual tile shape, and may be dynamically sized.
174static Value inferStride(Location loc, MemRefType mType, Value base,
175 RewriterBase &rewriter) {
176 assert(mType.getRank() >= 2 && "Invalid shape for AMX strides");
177 int64_t preLast = mType.getRank() - 2;
178 Type llvmInt64Type = rewriter.getIntegerType(width: 64);
179 unsigned width = mType.getElementType().getIntOrFloatBitWidth();
180 assert(llvm::isPowerOf2_64(width) && width >= 8);
181 unsigned bytes = width >> 3;
182 auto [strides, offset] = mType.getStridesAndOffset();
183 if (strides[preLast] == ShapedType::kDynamic) {
184 // Dynamic stride needs code to compute the stride at runtime.
185 MemRefDescriptor memrefDescriptor(base);
186 return computeStrideInBytes(
187 loc, mType, elementStride: memrefDescriptor.stride(builder&: rewriter, loc, pos: preLast), rewriter);
188 }
189 // Use direct constant for static stride.
190 auto attr = rewriter.getI64IntegerAttr(value: strides[preLast] * bytes);
191 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, value: attr)
192 .getResult();
193}
194
195LogicalResult x86::amx::TileZeroOp::verify() {
196 return verifyTileSize(op: *this, tp: getTileType());
197}
198
199SmallVector<Value> x86::amx::TileZeroOp::getIntrinsicOperands(
200 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
201 RewriterBase &rewriter) {
202 return getTileSizes(loc: getLoc(), tType: getTileType(), rewriter);
203}
204
205template <typename OpTy, typename = std::enable_if_t<
206 std::is_same_v<OpTy, x86::amx::TileLoadOp> ||
207 std::is_same_v<OpTy, x86::amx::TileStoreOp>>>
208static LogicalResult tileTransferVerifier(OpTy op) {
209 MemRefType memrefTy = op.getMemRefType();
210 unsigned rank = memrefTy.getRank();
211 if (op.getIndices().size() != rank)
212 return op.emitOpError("requires ") << rank << " indices";
213
214 if (failed(verifyTileSize(op, op.getTileType())))
215 return failure();
216
217 // Validate basic buffer properties when the stride is implicit.
218 if (!op.getStride()) {
219 if (rank < 2)
220 return op.emitOpError("requires at least 2D memref");
221 SmallVector<int64_t> strides;
222 int64_t offset;
223 if (failed(Result: memrefTy.getStridesAndOffset(strides, offset)) ||
224 strides.back() != 1)
225 return op.emitOpError("requires memref with unit innermost stride");
226 }
227
228 return success();
229}
230
231void x86::amx::TileLoadOp::build(OpBuilder &builder, OperationState &state,
232 Type res, Value base, ValueRange indices) {
233 build(odsBuilder&: builder, odsState&: state, res, base, indices, /*stride=*/nullptr);
234}
235
236LogicalResult x86::amx::TileLoadOp::verify() {
237 return tileTransferVerifier(op: *this);
238}
239
240SmallVector<Value> x86::amx::TileLoadOp::getIntrinsicOperands(
241 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
242 RewriterBase &rewriter) {
243 auto loc = getLoc();
244 Adaptor adaptor(operands, *this);
245
246 SmallVector<Value> intrinsicOperands;
247 intrinsicOperands.append(RHS: getTileSizes(loc, tType: getTileType(), rewriter));
248 intrinsicOperands.push_back(
249 Elt: LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: typeConverter, type: getMemRefType(),
250 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices()));
251 if (Value stride = adaptor.getStride())
252 intrinsicOperands.push_back(
253 Elt: computeStrideInBytes(loc, mType: getMemRefType(), elementStride: stride, rewriter));
254 else
255 intrinsicOperands.push_back(
256 Elt: inferStride(loc, mType: getMemRefType(), base: adaptor.getBase(), rewriter));
257
258 return intrinsicOperands;
259}
260
261void x86::amx::TileStoreOp::build(OpBuilder &builder, OperationState &state,
262 Value base, ValueRange indices, Value val) {
263 build(odsBuilder&: builder, odsState&: state, base, indices, val, /*stride=*/nullptr);
264}
265
266LogicalResult x86::amx::TileStoreOp::verify() {
267 return tileTransferVerifier(op: *this);
268}
269
270SmallVector<Value> x86::amx::TileStoreOp::getIntrinsicOperands(
271 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
272 RewriterBase &rewriter) {
273 auto loc = getLoc();
274 Adaptor adaptor(operands, *this);
275
276 SmallVector<Value> intrinsicOperands;
277 intrinsicOperands.append(RHS: getTileSizes(loc, tType: getTileType(), rewriter));
278 intrinsicOperands.push_back(
279 Elt: LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: typeConverter, type: getMemRefType(),
280 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices()));
281 if (Value stride = adaptor.getStride())
282 intrinsicOperands.push_back(
283 Elt: computeStrideInBytes(loc, mType: getMemRefType(), elementStride: stride, rewriter));
284 else
285 intrinsicOperands.push_back(
286 Elt: inferStride(loc, mType: getMemRefType(), base: adaptor.getBase(), rewriter));
287 intrinsicOperands.push_back(Elt: adaptor.getVal());
288
289 return intrinsicOperands;
290}
291
292LogicalResult x86::amx::TileMulFOp::verify() {
293 x86::amx::TileType aType = getLhsTileType();
294 x86::amx::TileType bType = getRhsTileType();
295 x86::amx::TileType cType = getTileType();
296 unsigned scale = 1;
297 if (aType.getElementType().isF8E4M3FN() || aType.getElementType().isF8E5M2())
298 scale = 2;
299 if (failed(Result: verifyTileSize(op: *this, tp: aType)) ||
300 failed(Result: verifyTileSize(op: *this, tp: bType)) ||
301 failed(Result: verifyTileSize(op: *this, tp: cType)) ||
302 failed(Result: verifyMultShape(op: *this, atp: aType, btp: bType, ctp: cType, scale)))
303 return failure();
304 Type ta = aType.getElementType();
305 Type tb = bType.getElementType();
306 Type tc = cType.getElementType();
307 bool flag1 = !ta.isBF16() && !ta.isF16() &&
308 !((ta.isF8E4M3FN() || ta.isF8E5M2()) &&
309 (tb.isF8E4M3FN() || tb.isF8E5M2()));
310 bool flag2 = (ta.isBF16() || ta.isF16()) ? (ta != tb) : false;
311 if (flag1 || flag2 || !tc.isF32())
312 return emitOpError(message: "unsupported type combination");
313 return success();
314}
315
316SmallVector<Value> x86::amx::TileMulFOp::getIntrinsicOperands(
317 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
318 RewriterBase &rewriter) {
319 auto loc = getLoc();
320 Adaptor adaptor(operands, *this);
321
322 x86::amx::TileType aType = getLhsTileType();
323 x86::amx::TileType bType = getRhsTileType();
324 SmallVector<Value> tsza = getTileSizes(loc, tType: aType, rewriter);
325 SmallVector<Value> tszb = getTileSizes(loc, tType: bType, rewriter);
326
327 SmallVector<Value> intrinsicOperands = {tsza[0], tszb[1],
328 tsza[1], adaptor.getAcc(),
329 adaptor.getLhs(), adaptor.getRhs()};
330
331 return intrinsicOperands;
332}
333
334LogicalResult x86::amx::TileMulIOp::verify() {
335 x86::amx::TileType aType = getLhsTileType();
336 x86::amx::TileType bType = getRhsTileType();
337 x86::amx::TileType cType = getTileType();
338 if (failed(Result: verifyTileSize(op: *this, tp: aType)) ||
339 failed(Result: verifyTileSize(op: *this, tp: bType)) ||
340 failed(Result: verifyTileSize(op: *this, tp: cType)) ||
341 failed(Result: verifyMultShape(op: *this, atp: aType, btp: bType, ctp: cType, scale: 2)))
342 return failure();
343 Type ta = aType.getElementType();
344 Type tb = bType.getElementType();
345 Type tc = cType.getElementType();
346 if (!ta.isInteger(width: 8) || !tb.isInteger(width: 8) || !tc.isInteger(width: 32))
347 return emitOpError(message: "unsupported type combination");
348 return success();
349}
350
351SmallVector<Value> x86::amx::TileMulIOp::getIntrinsicOperands(
352 ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter,
353 RewriterBase &rewriter) {
354 auto loc = getLoc();
355 Adaptor adaptor(operands, *this);
356
357 x86::amx::TileType aType = getLhsTileType();
358 x86::amx::TileType bType = getRhsTileType();
359 SmallVector<Value> tsza = getTileSizes(loc, tType: aType, rewriter);
360 SmallVector<Value> tszb = getTileSizes(loc, tType: bType, rewriter);
361
362 SmallVector<Value> intrinsicOperands = {tsza[0], tszb[1],
363 tsza[1], adaptor.getAcc(),
364 adaptor.getLhs(), adaptor.getRhs()};
365
366 return intrinsicOperands;
367}
368
369Type x86::amx::TileType::parse(AsmParser &parser) {
370 if (parser.parseLess())
371 return nullptr;
372
373 SmallVector<int64_t, 2> shape;
374 if (parser.parseDimensionList(dimensions&: shape, allowDynamic: false, withTrailingX: true))
375 return nullptr;
376
377 Type elementType;
378 if (parser.parseType(result&: elementType))
379 return nullptr;
380
381 if (parser.parseGreater())
382 return nullptr;
383
384 return AMXTileType::getChecked(
385 emitError: [&] { return parser.emitError(loc: parser.getNameLoc()); }, shape,
386 elementType);
387}
388
389void x86::amx::TileType::print(AsmPrinter &os) const {
390 os << "<";
391 os.printDimensionList(shape: getShape());
392 os << 'x';
393 os.printType(type: getElementType());
394 os << '>';
395}
396
397#define GET_OP_CLASSES
398#include "mlir/Dialect/X86/X86.cpp.inc"
399
400#define GET_TYPEDEF_CLASSES
401#include "mlir/Dialect/X86/X86Types.cpp.inc"
402