| 1 | //===- X86Dialect.cpp - MLIR X86 ops implementation -----------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the X86 dialect and its operations. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/X86/X86Dialect.h" |
| 14 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 18 | #include "mlir/IR/Builders.h" |
| 19 | #include "mlir/IR/OpImplementation.h" |
| 20 | #include "mlir/IR/TypeUtilities.h" |
| 21 | |
| 22 | using namespace mlir; |
| 23 | |
| 24 | #include "mlir/Dialect/X86/X86Interfaces.cpp.inc" |
| 25 | |
| 26 | #include "mlir/Dialect/X86/X86Dialect.cpp.inc" |
| 27 | |
| 28 | void x86::X86Dialect::initialize() { |
| 29 | addTypes< |
| 30 | #define GET_TYPEDEF_LIST |
| 31 | #include "mlir/Dialect/X86/X86Types.cpp.inc" |
| 32 | >(); |
| 33 | |
| 34 | addOperations< |
| 35 | #define GET_OP_LIST |
| 36 | #include "mlir/Dialect/X86/X86.cpp.inc" |
| 37 | >(); |
| 38 | } |
| 39 | |
| 40 | static Value getMemrefBuffPtr(Location loc, MemRefType type, Value buffer, |
| 41 | const LLVMTypeConverter &typeConverter, |
| 42 | RewriterBase &rewriter) { |
| 43 | MemRefDescriptor memRefDescriptor(buffer); |
| 44 | return memRefDescriptor.bufferPtr(builder&: rewriter, loc, converter: typeConverter, type); |
| 45 | } |
| 46 | |
| 47 | LogicalResult x86::avx512::MaskCompressOp::verify() { |
| 48 | if (getSrc() && getConstantSrc()) |
| 49 | return emitError(message: "cannot use both src and constant_src" ); |
| 50 | |
| 51 | if (getSrc() && (getSrc().getType() != getDst().getType())) |
| 52 | return emitError(message: "failed to verify that src and dst have same type" ); |
| 53 | |
| 54 | if (getConstantSrc() && (getConstantSrc()->getType() != getDst().getType())) |
| 55 | return emitError( |
| 56 | message: "failed to verify that constant_src and dst have same type" ); |
| 57 | |
| 58 | return success(); |
| 59 | } |
| 60 | |
| 61 | SmallVector<Value> x86::avx512::MaskCompressOp::getIntrinsicOperands( |
| 62 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 63 | RewriterBase &rewriter) { |
| 64 | auto loc = getLoc(); |
| 65 | Adaptor adaptor(operands, *this); |
| 66 | |
| 67 | auto opType = adaptor.getA().getType(); |
| 68 | Value src; |
| 69 | if (adaptor.getSrc()) { |
| 70 | src = adaptor.getSrc(); |
| 71 | } else if (adaptor.getConstantSrc()) { |
| 72 | src = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: opType, |
| 73 | value: adaptor.getConstantSrcAttr()); |
| 74 | } else { |
| 75 | auto zeroAttr = rewriter.getZeroAttr(type: opType); |
| 76 | src = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: opType, value: zeroAttr); |
| 77 | } |
| 78 | |
| 79 | return SmallVector<Value>{adaptor.getA(), src, adaptor.getK()}; |
| 80 | } |
| 81 | |
| 82 | SmallVector<Value> |
| 83 | x86::avx::DotOp::getIntrinsicOperands(ArrayRef<Value> operands, |
| 84 | const LLVMTypeConverter &typeConverter, |
| 85 | RewriterBase &rewriter) { |
| 86 | SmallVector<Value> intrinsicOperands(operands); |
| 87 | // Dot product of all elements, broadcasted to all elements. |
| 88 | Value scale = |
| 89 | LLVM::ConstantOp::create(builder&: rewriter, location: getLoc(), type: rewriter.getI8Type(), value: 0xff); |
| 90 | intrinsicOperands.push_back(Elt: scale); |
| 91 | |
| 92 | return intrinsicOperands; |
| 93 | } |
| 94 | |
| 95 | SmallVector<Value> x86::avx::BcstToPackedF32Op::getIntrinsicOperands( |
| 96 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 97 | RewriterBase &rewriter) { |
| 98 | Adaptor adaptor(operands, *this); |
| 99 | return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(), |
| 100 | typeConverter, rewriter)}; |
| 101 | } |
| 102 | |
| 103 | SmallVector<Value> x86::avx::CvtPackedEvenIndexedToF32Op::getIntrinsicOperands( |
| 104 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 105 | RewriterBase &rewriter) { |
| 106 | Adaptor adaptor(operands, *this); |
| 107 | return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(), |
| 108 | typeConverter, rewriter)}; |
| 109 | } |
| 110 | |
| 111 | SmallVector<Value> x86::avx::CvtPackedOddIndexedToF32Op::getIntrinsicOperands( |
| 112 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 113 | RewriterBase &rewriter) { |
| 114 | Adaptor adaptor(operands, *this); |
| 115 | return {getMemrefBuffPtr(loc: getLoc(), type: getA().getType(), buffer: adaptor.getA(), |
| 116 | typeConverter, rewriter)}; |
| 117 | } |
| 118 | |
| 119 | /// Verify that AMX supports the implied tile shape. |
| 120 | static LogicalResult verifyTileSize(Operation *op, x86::amx::TileType tp) { |
| 121 | const unsigned kMaxRows = 16; |
| 122 | const unsigned kBitsPerRow = 64 * 8; |
| 123 | unsigned col = tp.getDimSize(idx: 1) * tp.getElementType().getIntOrFloatBitWidth(); |
| 124 | if (tp.getDimSize(idx: 0) > kMaxRows) |
| 125 | return op->emitOpError(message: "bad row height: " ) << tp.getDimSize(idx: 0); |
| 126 | if (col > kBitsPerRow || col & 0x1f) |
| 127 | return op->emitOpError(message: "bad column width: " ) << (col >> 3); |
| 128 | return success(); |
| 129 | } |
| 130 | |
| 131 | /// Verify that AMX supports the multiplication. |
| 132 | static LogicalResult verifyMultShape(Operation *op, x86::amx::TileType atp, |
| 133 | x86::amx::TileType btp, |
| 134 | x86::amx::TileType ctp, unsigned scale) { |
| 135 | unsigned am = atp.getDimSize(idx: 0), ak = atp.getDimSize(idx: 1) >> scale; |
| 136 | unsigned bk = btp.getDimSize(idx: 0), bn = btp.getDimSize(idx: 1) >> scale; |
| 137 | unsigned cm = ctp.getDimSize(idx: 0), cn = ctp.getDimSize(idx: 1); |
| 138 | if (cm != am || cn != bn || ak != bk) |
| 139 | return op->emitOpError(message: "bad mult shape: " ) |
| 140 | << cm << " x " << cn << " x " << ak; |
| 141 | return success(); |
| 142 | } |
| 143 | |
| 144 | /// Maps the 2-dim vector shape to the two 16-bit tile sizes. The first |
| 145 | /// dimension directly translates into the number of rows of the tiles. |
| 146 | /// The second dimensions needs to be scaled by the number of bytes. |
| 147 | static SmallVector<Value> getTileSizes(Location loc, x86::amx::TileType tType, |
| 148 | RewriterBase &rewriter) { |
| 149 | Type llvmInt16Type = rewriter.getIntegerType(width: 16); |
| 150 | unsigned width = tType.getElementType().getIntOrFloatBitWidth(); |
| 151 | assert(llvm::isPowerOf2_64(width) && width >= 8); |
| 152 | unsigned bytes = width >> 3; |
| 153 | auto mattr = rewriter.getI16IntegerAttr(value: tType.getDimSize(idx: 0)); |
| 154 | auto nattr = rewriter.getI16IntegerAttr(value: tType.getDimSize(idx: 1) * bytes); |
| 155 | return SmallVector<Value>{ |
| 156 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt16Type, value: mattr), |
| 157 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt16Type, value: nattr)}; |
| 158 | } |
| 159 | |
| 160 | /// Returns stride expressed in number of bytes for the given `elementStride` |
| 161 | /// stride encoded in number of elements of the type `mType`. |
| 162 | static Value computeStrideInBytes(Location loc, MemRefType mType, |
| 163 | Value elementStride, RewriterBase &rewriter) { |
| 164 | Type llvmInt64Type = rewriter.getIntegerType(width: 64); |
| 165 | unsigned bytes = mType.getElementType().getIntOrFloatBitWidth() / 8; |
| 166 | auto attr = rewriter.getI64IntegerAttr(value: bytes); |
| 167 | Value scale = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, value: attr); |
| 168 | return LLVM::MulOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, lhs: scale, rhs: elementStride) |
| 169 | .getResult(); |
| 170 | } |
| 171 | |
| 172 | /// Maps the 2-dim memref shape to the 64-bit stride. Note that the buffer |
| 173 | /// shape may "envelop" the actual tile shape, and may be dynamically sized. |
| 174 | static Value inferStride(Location loc, MemRefType mType, Value base, |
| 175 | RewriterBase &rewriter) { |
| 176 | assert(mType.getRank() >= 2 && "Invalid shape for AMX strides" ); |
| 177 | int64_t preLast = mType.getRank() - 2; |
| 178 | Type llvmInt64Type = rewriter.getIntegerType(width: 64); |
| 179 | unsigned width = mType.getElementType().getIntOrFloatBitWidth(); |
| 180 | assert(llvm::isPowerOf2_64(width) && width >= 8); |
| 181 | unsigned bytes = width >> 3; |
| 182 | auto [strides, offset] = mType.getStridesAndOffset(); |
| 183 | if (strides[preLast] == ShapedType::kDynamic) { |
| 184 | // Dynamic stride needs code to compute the stride at runtime. |
| 185 | MemRefDescriptor memrefDescriptor(base); |
| 186 | return computeStrideInBytes( |
| 187 | loc, mType, elementStride: memrefDescriptor.stride(builder&: rewriter, loc, pos: preLast), rewriter); |
| 188 | } |
| 189 | // Use direct constant for static stride. |
| 190 | auto attr = rewriter.getI64IntegerAttr(value: strides[preLast] * bytes); |
| 191 | return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmInt64Type, value: attr) |
| 192 | .getResult(); |
| 193 | } |
| 194 | |
| 195 | LogicalResult x86::amx::TileZeroOp::verify() { |
| 196 | return verifyTileSize(op: *this, tp: getTileType()); |
| 197 | } |
| 198 | |
| 199 | SmallVector<Value> x86::amx::TileZeroOp::getIntrinsicOperands( |
| 200 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 201 | RewriterBase &rewriter) { |
| 202 | return getTileSizes(loc: getLoc(), tType: getTileType(), rewriter); |
| 203 | } |
| 204 | |
| 205 | template <typename OpTy, typename = std::enable_if_t< |
| 206 | std::is_same_v<OpTy, x86::amx::TileLoadOp> || |
| 207 | std::is_same_v<OpTy, x86::amx::TileStoreOp>>> |
| 208 | static LogicalResult tileTransferVerifier(OpTy op) { |
| 209 | MemRefType memrefTy = op.getMemRefType(); |
| 210 | unsigned rank = memrefTy.getRank(); |
| 211 | if (op.getIndices().size() != rank) |
| 212 | return op.emitOpError("requires " ) << rank << " indices" ; |
| 213 | |
| 214 | if (failed(verifyTileSize(op, op.getTileType()))) |
| 215 | return failure(); |
| 216 | |
| 217 | // Validate basic buffer properties when the stride is implicit. |
| 218 | if (!op.getStride()) { |
| 219 | if (rank < 2) |
| 220 | return op.emitOpError("requires at least 2D memref" ); |
| 221 | SmallVector<int64_t> strides; |
| 222 | int64_t offset; |
| 223 | if (failed(Result: memrefTy.getStridesAndOffset(strides, offset)) || |
| 224 | strides.back() != 1) |
| 225 | return op.emitOpError("requires memref with unit innermost stride" ); |
| 226 | } |
| 227 | |
| 228 | return success(); |
| 229 | } |
| 230 | |
| 231 | void x86::amx::TileLoadOp::build(OpBuilder &builder, OperationState &state, |
| 232 | Type res, Value base, ValueRange indices) { |
| 233 | build(odsBuilder&: builder, odsState&: state, res, base, indices, /*stride=*/nullptr); |
| 234 | } |
| 235 | |
| 236 | LogicalResult x86::amx::TileLoadOp::verify() { |
| 237 | return tileTransferVerifier(op: *this); |
| 238 | } |
| 239 | |
| 240 | SmallVector<Value> x86::amx::TileLoadOp::getIntrinsicOperands( |
| 241 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 242 | RewriterBase &rewriter) { |
| 243 | auto loc = getLoc(); |
| 244 | Adaptor adaptor(operands, *this); |
| 245 | |
| 246 | SmallVector<Value> intrinsicOperands; |
| 247 | intrinsicOperands.append(RHS: getTileSizes(loc, tType: getTileType(), rewriter)); |
| 248 | intrinsicOperands.push_back( |
| 249 | Elt: LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: typeConverter, type: getMemRefType(), |
| 250 | memRefDesc: adaptor.getBase(), indices: adaptor.getIndices())); |
| 251 | if (Value stride = adaptor.getStride()) |
| 252 | intrinsicOperands.push_back( |
| 253 | Elt: computeStrideInBytes(loc, mType: getMemRefType(), elementStride: stride, rewriter)); |
| 254 | else |
| 255 | intrinsicOperands.push_back( |
| 256 | Elt: inferStride(loc, mType: getMemRefType(), base: adaptor.getBase(), rewriter)); |
| 257 | |
| 258 | return intrinsicOperands; |
| 259 | } |
| 260 | |
| 261 | void x86::amx::TileStoreOp::build(OpBuilder &builder, OperationState &state, |
| 262 | Value base, ValueRange indices, Value val) { |
| 263 | build(odsBuilder&: builder, odsState&: state, base, indices, val, /*stride=*/nullptr); |
| 264 | } |
| 265 | |
| 266 | LogicalResult x86::amx::TileStoreOp::verify() { |
| 267 | return tileTransferVerifier(op: *this); |
| 268 | } |
| 269 | |
| 270 | SmallVector<Value> x86::amx::TileStoreOp::getIntrinsicOperands( |
| 271 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 272 | RewriterBase &rewriter) { |
| 273 | auto loc = getLoc(); |
| 274 | Adaptor adaptor(operands, *this); |
| 275 | |
| 276 | SmallVector<Value> intrinsicOperands; |
| 277 | intrinsicOperands.append(RHS: getTileSizes(loc, tType: getTileType(), rewriter)); |
| 278 | intrinsicOperands.push_back( |
| 279 | Elt: LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: typeConverter, type: getMemRefType(), |
| 280 | memRefDesc: adaptor.getBase(), indices: adaptor.getIndices())); |
| 281 | if (Value stride = adaptor.getStride()) |
| 282 | intrinsicOperands.push_back( |
| 283 | Elt: computeStrideInBytes(loc, mType: getMemRefType(), elementStride: stride, rewriter)); |
| 284 | else |
| 285 | intrinsicOperands.push_back( |
| 286 | Elt: inferStride(loc, mType: getMemRefType(), base: adaptor.getBase(), rewriter)); |
| 287 | intrinsicOperands.push_back(Elt: adaptor.getVal()); |
| 288 | |
| 289 | return intrinsicOperands; |
| 290 | } |
| 291 | |
| 292 | LogicalResult x86::amx::TileMulFOp::verify() { |
| 293 | x86::amx::TileType aType = getLhsTileType(); |
| 294 | x86::amx::TileType bType = getRhsTileType(); |
| 295 | x86::amx::TileType cType = getTileType(); |
| 296 | unsigned scale = 1; |
| 297 | if (aType.getElementType().isF8E4M3FN() || aType.getElementType().isF8E5M2()) |
| 298 | scale = 2; |
| 299 | if (failed(Result: verifyTileSize(op: *this, tp: aType)) || |
| 300 | failed(Result: verifyTileSize(op: *this, tp: bType)) || |
| 301 | failed(Result: verifyTileSize(op: *this, tp: cType)) || |
| 302 | failed(Result: verifyMultShape(op: *this, atp: aType, btp: bType, ctp: cType, scale))) |
| 303 | return failure(); |
| 304 | Type ta = aType.getElementType(); |
| 305 | Type tb = bType.getElementType(); |
| 306 | Type tc = cType.getElementType(); |
| 307 | bool flag1 = !ta.isBF16() && !ta.isF16() && |
| 308 | !((ta.isF8E4M3FN() || ta.isF8E5M2()) && |
| 309 | (tb.isF8E4M3FN() || tb.isF8E5M2())); |
| 310 | bool flag2 = (ta.isBF16() || ta.isF16()) ? (ta != tb) : false; |
| 311 | if (flag1 || flag2 || !tc.isF32()) |
| 312 | return emitOpError(message: "unsupported type combination" ); |
| 313 | return success(); |
| 314 | } |
| 315 | |
| 316 | SmallVector<Value> x86::amx::TileMulFOp::getIntrinsicOperands( |
| 317 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 318 | RewriterBase &rewriter) { |
| 319 | auto loc = getLoc(); |
| 320 | Adaptor adaptor(operands, *this); |
| 321 | |
| 322 | x86::amx::TileType aType = getLhsTileType(); |
| 323 | x86::amx::TileType bType = getRhsTileType(); |
| 324 | SmallVector<Value> tsza = getTileSizes(loc, tType: aType, rewriter); |
| 325 | SmallVector<Value> tszb = getTileSizes(loc, tType: bType, rewriter); |
| 326 | |
| 327 | SmallVector<Value> intrinsicOperands = {tsza[0], tszb[1], |
| 328 | tsza[1], adaptor.getAcc(), |
| 329 | adaptor.getLhs(), adaptor.getRhs()}; |
| 330 | |
| 331 | return intrinsicOperands; |
| 332 | } |
| 333 | |
| 334 | LogicalResult x86::amx::TileMulIOp::verify() { |
| 335 | x86::amx::TileType aType = getLhsTileType(); |
| 336 | x86::amx::TileType bType = getRhsTileType(); |
| 337 | x86::amx::TileType cType = getTileType(); |
| 338 | if (failed(Result: verifyTileSize(op: *this, tp: aType)) || |
| 339 | failed(Result: verifyTileSize(op: *this, tp: bType)) || |
| 340 | failed(Result: verifyTileSize(op: *this, tp: cType)) || |
| 341 | failed(Result: verifyMultShape(op: *this, atp: aType, btp: bType, ctp: cType, scale: 2))) |
| 342 | return failure(); |
| 343 | Type ta = aType.getElementType(); |
| 344 | Type tb = bType.getElementType(); |
| 345 | Type tc = cType.getElementType(); |
| 346 | if (!ta.isInteger(width: 8) || !tb.isInteger(width: 8) || !tc.isInteger(width: 32)) |
| 347 | return emitOpError(message: "unsupported type combination" ); |
| 348 | return success(); |
| 349 | } |
| 350 | |
| 351 | SmallVector<Value> x86::amx::TileMulIOp::getIntrinsicOperands( |
| 352 | ArrayRef<Value> operands, const LLVMTypeConverter &typeConverter, |
| 353 | RewriterBase &rewriter) { |
| 354 | auto loc = getLoc(); |
| 355 | Adaptor adaptor(operands, *this); |
| 356 | |
| 357 | x86::amx::TileType aType = getLhsTileType(); |
| 358 | x86::amx::TileType bType = getRhsTileType(); |
| 359 | SmallVector<Value> tsza = getTileSizes(loc, tType: aType, rewriter); |
| 360 | SmallVector<Value> tszb = getTileSizes(loc, tType: bType, rewriter); |
| 361 | |
| 362 | SmallVector<Value> intrinsicOperands = {tsza[0], tszb[1], |
| 363 | tsza[1], adaptor.getAcc(), |
| 364 | adaptor.getLhs(), adaptor.getRhs()}; |
| 365 | |
| 366 | return intrinsicOperands; |
| 367 | } |
| 368 | |
| 369 | Type x86::amx::TileType::parse(AsmParser &parser) { |
| 370 | if (parser.parseLess()) |
| 371 | return nullptr; |
| 372 | |
| 373 | SmallVector<int64_t, 2> shape; |
| 374 | if (parser.parseDimensionList(dimensions&: shape, allowDynamic: false, withTrailingX: true)) |
| 375 | return nullptr; |
| 376 | |
| 377 | Type elementType; |
| 378 | if (parser.parseType(result&: elementType)) |
| 379 | return nullptr; |
| 380 | |
| 381 | if (parser.parseGreater()) |
| 382 | return nullptr; |
| 383 | |
| 384 | return AMXTileType::getChecked( |
| 385 | emitError: [&] { return parser.emitError(loc: parser.getNameLoc()); }, shape, |
| 386 | elementType); |
| 387 | } |
| 388 | |
| 389 | void x86::amx::TileType::print(AsmPrinter &os) const { |
| 390 | os << "<" ; |
| 391 | os.printDimensionList(shape: getShape()); |
| 392 | os << 'x'; |
| 393 | os.printType(type: getElementType()); |
| 394 | os << '>'; |
| 395 | } |
| 396 | |
| 397 | #define GET_OP_CLASSES |
| 398 | #include "mlir/Dialect/X86/X86.cpp.inc" |
| 399 | |
| 400 | #define GET_TYPEDEF_CLASSES |
| 401 | #include "mlir/Dialect/X86/X86Types.cpp.inc" |
| 402 | |