1//===- NVGPUDialect.cpp - MLIR NVGPU ops implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the NVGPU dialect and its operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
14#include "mlir/Dialect/GPU/IR/GPUDialect.h"
15#include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h"
16#include "mlir/IR/Builders.h"
17#include "mlir/IR/BuiltinAttributes.h"
18#include "mlir/IR/BuiltinTypes.h"
19#include "mlir/IR/Diagnostics.h"
20#include "mlir/IR/TypeUtilities.h"
21#include "mlir/IR/Verifier.h"
22#include "llvm/ADT/STLExtras.h"
23
24using namespace mlir;
25using namespace mlir::nvgpu;
26
27#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.cpp.inc"
28
29void NVGPUDialect::initialize() {
30 addTypes<
31#define GET_TYPEDEF_LIST
32#include "mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.cpp.inc"
33 >();
34 addAttributes<
35#define GET_ATTRDEF_LIST
36#include "mlir/Dialect/NVGPU/IR/NVGPUAttrDefs.cpp.inc"
37 >();
38 addOperations<
39#define GET_OP_LIST
40#include "mlir/Dialect/NVGPU/IR/NVGPUOps.cpp.inc"
41 >();
42 declarePromisedInterfaces<memref::IndexedAccessOpInterface, LdMatrixOp>();
43 declarePromisedInterfaces<memref::IndexedMemCopyOpInterface,
44 DeviceAsyncCopyOp>();
45}
46
47bool NVGPUDialect::isSharedMemoryAddressSpace(Attribute memorySpace) {
48 if (!memorySpace)
49 return false;
50 if (auto intAttr = llvm::dyn_cast<IntegerAttr>(Val&: memorySpace))
51 return intAttr.getValue().getZExtValue() ==
52 NVGPUDialect::kSharedMemoryAddressSpace;
53 if (auto gpuAttr = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memorySpace))
54 return gpuAttr.getValue() == gpu::AddressSpace::Workgroup;
55 return false;
56}
57
58bool NVGPUDialect::hasSharedMemoryAddressSpace(MemRefType type) {
59 Attribute memorySpace = type.getMemorySpace();
60 return isSharedMemoryAddressSpace(memorySpace);
61}
62
63//===----------------------------------------------------------------------===//
64// NVGPU_DeviceAsyncCopyOp
65//===----------------------------------------------------------------------===//
66
67LogicalResult DeviceAsyncCopyOp::verify() {
68 auto srcMemref = llvm::cast<MemRefType>(Val: getSrc().getType());
69 auto dstMemref = llvm::cast<MemRefType>(Val: getDst().getType());
70
71 if (!srcMemref.isLastDimUnitStride())
72 return emitError(message: "source memref most minor dim must have unit stride");
73 if (!dstMemref.isLastDimUnitStride())
74 return emitError(message: "destination memref most minor dim must have unit stride");
75 if (!NVGPUDialect::hasSharedMemoryAddressSpace(type: dstMemref))
76 return emitError()
77 << "destination memref must have a memory space attribute of "
78 "IntegerAttr("
79 << NVGPUDialect::kSharedMemoryAddressSpace
80 << ") or gpu::AddressSpaceAttr(Workgroup)";
81 if (dstMemref.getElementType() != srcMemref.getElementType())
82 return emitError(message: "source and destination must have the same element type");
83 if (size_t(srcMemref.getRank()) != getSrcIndices().size())
84 return emitOpError() << "expected " << srcMemref.getRank()
85 << " source indices, got " << getSrcIndices().size();
86 if (size_t(dstMemref.getRank()) != getDstIndices().size())
87 return emitOpError() << "expected " << dstMemref.getRank()
88 << " destination indices, got "
89 << getDstIndices().size();
90 int64_t dstElements = getDstElements().getZExtValue();
91 int64_t sizeInBytes = (dstMemref.getElementTypeBitWidth() * dstElements) / 8;
92 if (sizeInBytes != 4 && sizeInBytes != 8 && sizeInBytes != 16) {
93 unsigned dstWidth = dstMemref.getElementTypeBitWidth();
94 InFlightDiagnostic diag = emitError();
95 diag << "Requested copy elements is " << dstElements << " with width "
96 << dstMemref.getElementTypeBitWidth()
97 << ". But copy elements could be one of ";
98 if ((32 / dstWidth) > 0)
99 diag << (32 / dstWidth) << ", ";
100 if ((64 / dstWidth) > 0)
101 diag << (64 / dstWidth) << ", ";
102 if ((128 / dstWidth) > 0)
103 diag << (128 / dstWidth) << ".";
104 return diag;
105 }
106 if (getBypassL1().has_value()) {
107 int64_t req = 16 * 8 / dstMemref.getElementTypeBitWidth();
108 if (getBypassL1().value() && sizeInBytes != 16) {
109 return emitOpError() << "bypassL1 does not satify alignment for "
110 << dstMemref << " with destination element "
111 << dstElements
112 << ". Unset bypassL1, or set "
113 "destination element to "
114 << req;
115 }
116 }
117 return success();
118}
119
120//===----------------------------------------------------------------------===//
121// NVGPU_MmaSyncOp
122//===----------------------------------------------------------------------===//
123void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder,
124 ::mlir::OperationState &odsState, Value matrixA,
125 Value matrixB, Value matrixC, ArrayAttr mmaShape) {
126 build(odsBuilder, odsState, res: matrixC.getType(), matrixA, matrixB, matrixC,
127 mmaShape, tf32Enabled: UnitAttr());
128}
129
130void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder,
131 ::mlir::OperationState &odsState, Value matrixA,
132 Value matrixB, Value matrixC, ArrayRef<int64_t> mmaShape,
133 bool tf32Enabled) {
134 build(odsBuilder, odsState, res: matrixC.getType(), matrixA, matrixB, matrixC,
135 mmaShape: odsBuilder.getI64ArrayAttr(values: mmaShape),
136 tf32Enabled: tf32Enabled ? odsBuilder.getUnitAttr() : UnitAttr());
137}
138
139/// Performs verification for MmaSyncOp and MmaSparseSyncOp.
140static LogicalResult verifyMmaSyncOp(Operation *op,
141 TypedValue<VectorType> matrixA,
142 TypedValue<VectorType> matrixB,
143 TypedValue<VectorType> matrixC,
144 const std::array<int64_t, 3> &mmaShape,
145 bool tf32Enabled, bool sparse = false) {
146 // The verification for mma.sync covering various shapes and data types is
147 // based on the fundamental tensor core shape.
148
149 // "Fundamental" tensor core shapes:
150 // - For F32 (TF32), F16, S8, and S4 data
151 // types the fundamental tensor core operation is of shape 8-by-8-by-128b.
152 // - F64 is an exception and is of shape 8-by-8-by-256b.
153 int64_t shapeM = 8;
154 int64_t shapeN = 8;
155 int64_t shapeK; // set based on data type (128b for all data types except F64)
156
157 // Number of elements A, B, and C per thread per fundamental tensor core tile
158 int64_t numElementA; // set based on data type (32b except F64)
159 int64_t numElementB; // set based on data type (32b except F64)
160 int64_t numElementC{2}; // two accumulator elements per fundamental tile
161
162 // nvgpu.mma.sync vector operands (per thread)
163 auto aVector = matrixA.getType();
164 auto bVector = matrixB.getType();
165 auto cVector = matrixC.getType();
166
167 // vector shapes
168 ArrayRef<int64_t> aShape = aVector.getShape();
169 ArrayRef<int64_t> bShape = bVector.getShape();
170 ArrayRef<int64_t> cShape = cVector.getShape();
171
172 // vector element type
173 Type aType = aVector.getElementType();
174
175 // Certain data types are not allowed in sparse mode.
176 if (sparse && aType.isF64())
177 return op->emitError() << "f64 is not supported for sparse mode";
178
179 if (aType.isF64()) {
180 // exception to 8-by-8-128b fundamental tensor core tile size
181 shapeK = 4;
182 numElementA = 1;
183 numElementB = 1;
184 } else if (aType.isF32() || aType.isBF16() || aType.isF16() ||
185 aType.isInteger(width: 8) || aType.isInteger(width: 4) || aType.isF8E4M3FN() ||
186 aType.isF8E5M2()) {
187 // 8-by-8-128b fundamental tensor core tile size
188 int operandBitwidth = aType.getIntOrFloatBitWidth();
189 shapeK = 128 / operandBitwidth; // 128b wide shapeK
190
191 numElementA = 32 / operandBitwidth; // 32b wide operand A
192 numElementB = 32 / operandBitwidth; // 32b wide operand B
193 } else {
194 return op->emitError()
195 << "expected input data type (i4,i8,f16,bf16,tf32,f64,"
196 "f8E4M3FN,f8E5M2) supported by "
197 << op->getName();
198 }
199
200 //
201 // Basic verification
202 //
203
204 if (aShape.size() != 2) {
205 return op->emitError() << "matrixA must be 2 dimensional vector";
206 }
207
208 if (bShape.size() != 2) {
209 return op->emitError() << "matrixB must be 2 dimensional vector";
210 }
211
212 if (cShape.size() != 2) {
213 return op->emitError() << "matrixC must be 2 dimensional vector";
214 }
215
216 auto [m, n, k] = mmaShape;
217
218 // verify warp-wide size for vector a
219 int64_t sparseFactor = sparse ? 2 : 1;
220 if (aShape[0] * aShape[1] * kWarpSize != m * k / sparseFactor)
221 return op->emitOpError()
222 << "expected " << m * k << " warp-wide matrix A elements";
223
224 // verify warp-wide size for vector b
225 if (bShape[0] * bShape[1] * kWarpSize != k * n)
226 return op->emitOpError()
227 << "expected " << k * n << " warp-wide matrix B elements";
228
229 // verify warp-wide size for vector c
230 if (cShape[0] * cShape[1] * kWarpSize != m * n)
231 return op->emitOpError()
232 << "expected " << m * n << " warp-wide matrix C elements";
233
234 // verify tf32 tensor cores are enabled for only F32 datatype
235 if (tf32Enabled && !(aType.isF32()))
236 return op->emitOpError()
237 << "expected tf32 tensor cores only for F32 operands";
238
239 //
240 // Extended verification
241 //
242
243 // tiles of fundamental tensor core operations
244 int64_t mTile = m / shapeM;
245 int64_t nTile = n / shapeN;
246 int64_t kTile = k / shapeK;
247
248 // verify shape of aVector
249 if ((aShape[0] != mTile * kTile / (sparse ? 2 : 1)) ||
250 (aShape[1] != numElementA))
251 return op->emitOpError() << "expected matrix A to be shaped ("
252 << mTile * kTile << " x " << numElementA << ")";
253
254 // verify shape of bVector
255 if ((bShape[0] != kTile * nTile) || (bShape[1] != numElementB))
256 return op->emitOpError() << "expected matrix B to be shaped ("
257 << kTile * nTile << " x " << numElementB << ")";
258
259 // verify shape of cVector
260 if ((cShape[0] != mTile * nTile) || (cShape[1] != numElementC))
261 return op->emitOpError() << "expected matrix C to be shaped ("
262 << mTile * nTile << " x " << numElementC << ")";
263
264 return success();
265}
266
267LogicalResult MmaSyncOp::verify() {
268 if (getMmaShape().size() != 3)
269 return emitOpError() << "mmaShape must have exactly 3 elements";
270
271 return verifyMmaSyncOp(op: this->getOperation(), matrixA: getMatrixA(), matrixB: getMatrixB(),
272 matrixC: getMatrixC(), mmaShape: getMmaShapeAsArray(),
273 tf32Enabled: getTf32Enabled().value_or(u: false));
274}
275
276//===----------------------------------------------------------------------===//
277// NVGPU_MmaSparseSyncOp
278//===----------------------------------------------------------------------===//
279void MmaSparseSyncOp::build(::mlir::OpBuilder &odsBuilder,
280 ::mlir::OperationState &odsState, Value matrixA,
281 Value matrixB, Value matrixC, Value sparseMetadata,
282 ArrayRef<int64_t> mmaShape) {
283 build(odsBuilder, odsState, res: matrixC.getType(), matrixA, matrixB, matrixC,
284 sparseMetadata, mmaShape: odsBuilder.getI64ArrayAttr(values: mmaShape), sparsitySelector: 0, tf32Enabled: UnitAttr());
285}
286
287LogicalResult MmaSparseSyncOp::verify() {
288 unsigned sparsitySelector = getSparsitySelector();
289 if (sparsitySelector > 1)
290 return emitOpError() << "sparsity selector should be 0 or 1";
291
292 if (getMmaShape().size() != 3)
293 return emitOpError() << "mmaShape must have exactly 3 elements";
294
295 return verifyMmaSyncOp(op: this->getOperation(), matrixA: getMatrixA(), matrixB: getMatrixB(),
296 matrixC: getMatrixC(), mmaShape: getMmaShapeAsArray(),
297 tf32Enabled: getTf32Enabled().value_or(u: false), sparse: true);
298}
299
300//===----------------------------------------------------------------------===//
301// NVGPU_LdMatrixOp
302//===----------------------------------------------------------------------===//
303LogicalResult LdMatrixOp::verify() {
304 // ldmatrix reads data from source in shared memory
305 auto srcMemref = llvm::cast<MemRefType>(Val: getSrcMemref().getType());
306
307 // ldmatrix writes data to result/destination in vector registers
308 auto resVector = llvm::cast<VectorType>(Val: getRes().getType());
309
310 // vector register shape, element type, and bitwidth
311 ArrayRef<int64_t> resShape = resVector.getShape();
312 Type resType = resVector.getElementType();
313 int64_t elementBitWidth = resType.getIntOrFloatBitWidth();
314
315 // ldmatrix loads 32 bits into vector registers per 8-by-8 tile per thread
316 int64_t numElementsPer32b = 32 / elementBitWidth;
317
318 // number of 8-by-8 tiles
319 int64_t numTiles = getNumTiles();
320
321 // transpose elements in vector registers at 16b granularity when true
322 bool isTranspose = getTranspose();
323
324 //
325 // verification
326 //
327
328 if (!NVGPUDialect::hasSharedMemoryAddressSpace(type: srcMemref))
329 return emitError()
330 << "expected nvgpu.ldmatrix srcMemref must have a memory space "
331 "attribute of IntegerAttr("
332 << NVGPUDialect::kSharedMemoryAddressSpace
333 << ") or gpu::AddressSpaceAttr(Workgroup)";
334 if (elementBitWidth > 32)
335 return emitError() << "nvgpu.ldmatrix works for 32b or lower";
336 if (isTranspose && !(elementBitWidth == 16))
337 return emitError()
338 << "nvgpu.ldmatrix transpose works only at 16b granularity";
339 if (resShape.size() != 2) {
340 return emitError() << "results must be 2 dimensional vector";
341 }
342 if (!(resShape[1] == numElementsPer32b))
343 return emitError() << "expected vector register shape[1] = "
344 << numElementsPer32b;
345 if (!(resShape[0] == numTiles))
346 return emitError()
347 << "expected vector register shape[0] and numTiles to match";
348
349 return success();
350}
351
352//===----------------------------------------------------------------------===//
353// NVGPU_TmaAsyncLoadOp
354//===----------------------------------------------------------------------===//
355
356static unsigned getSwizzleBytes(TensorMapSwizzleKind kind) {
357 switch (kind) {
358 case TensorMapSwizzleKind::SWIZZLE_32B:
359 return 32;
360 case TensorMapSwizzleKind::SWIZZLE_64B:
361 return 64;
362 case TensorMapSwizzleKind::SWIZZLE_128B:
363 return 128;
364 default:
365 return 0;
366 }
367}
368
369std::optional<InFlightDiagnostic> verifyTmaDescriptorWithMemref(
370 Operation *op, TensorMapDescriptorType descType,
371 std::optional<MemRefType> memrefType = std::nullopt) {
372 MemRefType descMemref = descType.getTensor();
373 // Limitation
374 if (descType.getInterleave() != TensorMapInterleaveKind::INTERLEAVE_NONE)
375 return op->emitError() << "Interleave options are not supported yet.";
376
377 // Address space check for shared memory check
378 if (!NVGPUDialect::hasSharedMemoryAddressSpace(type: descMemref)) {
379 return op->emitError() << "the tensor map descriptor has incorrect address "
380 "space, it must be shared memory address space.";
381 }
382 // Support only static shape for the time being
383 if (!descMemref.hasStaticShape())
384 return op->emitError() << "the tensor map descriptor must be static shaped";
385
386 for (auto dim : descMemref.getShape()) {
387 if (dim <= 0 || dim > kMaxTMADimension) {
388 return op->emitError() << "the tensor map descriptor must have "
389 "dimensions between 1 and "
390 << kMaxTMADimension << " but it is " << dim;
391 }
392 }
393 if (descMemref.getRank() > 1 &&
394 descType.getSwizzle() != TensorMapSwizzleKind::SWIZZLE_NONE) {
395 unsigned lastDimensionByte =
396 descMemref.getElementTypeBitWidth() * descMemref.getShape().back() / 8;
397 unsigned expectByte = getSwizzleBytes(kind: descType.getSwizzle());
398 if (lastDimensionByte != expectByte)
399 return op->emitError() << "the tensormap descriptor must have last "
400 "dimension of "
401 << expectByte << " bytes but it is "
402 << lastDimensionByte << " bytes";
403 }
404
405 // No verification if memref type is not provided
406 if (!memrefType.has_value())
407 return std::nullopt;
408
409 MemRefType dstMemref = memrefType.value();
410
411 // Check element type
412 if (descMemref.getElementType() != dstMemref.getElementType()) {
413 return op->emitError() << "the element type of tensor map descriptor and "
414 "memref must be same";
415 }
416
417 if (!NVGPUDialect::hasSharedMemoryAddressSpace(type: dstMemref)) {
418 return op->emitError() << "the destination memref has incorrect address "
419 "space, it must be shared memory address space.";
420 }
421 if (!dstMemref.hasStaticShape())
422 return op->emitError() << "the destination memref must be static shaped";
423
424 if (dstMemref.getRank() != descMemref.getRank()) {
425 return op->emitError() << "the shape of tensor map descriptor and "
426 "memref must have same rank";
427 }
428 if (!descMemref.getShape().equals(RHS: dstMemref.getShape())) {
429 return op->emitError() << "memref and tensor map shapes mismatch "
430 << descMemref << " != " << dstMemref;
431 }
432
433 int lastDimBytes =
434 descMemref.getShape().back() * descMemref.getElementTypeBitWidth() / 8;
435 if (lastDimBytes % kTMALastdimByte != 0) {
436 return op->emitError() << "the bytes in the last dimension of the tensor "
437 "map must be a multiple of 16";
438 }
439 return std::nullopt;
440}
441
442LogicalResult TmaAsyncLoadOp::verify() {
443 std::optional<InFlightDiagnostic> error = verifyTmaDescriptorWithMemref(
444 op: *this, descType: getTensorMapDescriptor().getType(), memrefType: getDst().getType());
445 if (error.has_value())
446 return error.value();
447
448 if (getCoordinates().size() > kMaxTMATensorDimension) {
449 return emitError() << "Maximum " << kMaxTMATensorDimension
450 << " coordinates are supported.";
451 }
452 if (getCoordinates().size() !=
453 size_t(getTensorMapDescriptor().getType().getTensor().getRank())) {
454 return emitError() << "number of coordinates do not match with the rank of "
455 "tensor descriptor map.";
456 }
457
458 return success();
459}
460
461//===----------------------------------------------------------------------===//
462// NVGPU_TmaAsyncStoreOp
463//===----------------------------------------------------------------------===//
464
465LogicalResult TmaAsyncStoreOp::verify() {
466 std::optional<InFlightDiagnostic> error = verifyTmaDescriptorWithMemref(
467 op: *this, descType: getTensorMapDescriptor().getType(), memrefType: getSrc().getType());
468 if (error.has_value())
469 return error.value();
470
471 if (getCoordinates().size() > kMaxTMATensorDimension) {
472 return emitError() << "Maximum " << kMaxTMATensorDimension
473 << " coordinates are supported.";
474 }
475 if (getCoordinates().size() !=
476 size_t(getTensorMapDescriptor().getType().getTensor().getRank())) {
477 return emitError() << "number of coordinates do not match with the rank of "
478 "tensor descriptor map.";
479 }
480
481 return success();
482}
483
484LogicalResult TmaCreateDescriptorOp::verify() {
485 if (getBoxDimensions().size() > kMaxTMATensorDimension) {
486 return emitError() << "Maximum " << kMaxTMATensorDimension
487 << " coordinates are supported.";
488 }
489
490 std::optional<InFlightDiagnostic> error =
491 verifyTmaDescriptorWithMemref(op: *this, descType: getTensorMap().getType());
492 if (error.has_value())
493 return error.value();
494
495 return success();
496}
497
498//===----------------------------------------------------------------------===//
499// NVGPU_WarpgroupGenerateDescriptorOp
500//===----------------------------------------------------------------------===//
501
502LogicalResult WarpgroupGenerateDescriptorOp::verify() {
503 std::optional<InFlightDiagnostic> error =
504 verifyTmaDescriptorWithMemref(op: *this, descType: getTensorMap().getType());
505 if (error.has_value())
506 return error.value();
507
508 if (getTensorMap().getType().getSwizzle() !=
509 TensorMapSwizzleKind::SWIZZLE_128B) {
510 return emitError() << "supports only "
511 << stringifyTensorMapSwizzleKind(
512 TensorMapSwizzleKind::SWIZZLE_128B)
513 << " is supported for the time being";
514 }
515
516 if (getTensorMap().getType().getInterleave() !=
517 TensorMapInterleaveKind::INTERLEAVE_NONE) {
518 return emitError() << "supports only "
519 << stringifyTensorMapInterleaveKind(
520 TensorMapInterleaveKind::INTERLEAVE_NONE)
521 << " is supported for the time being";
522 }
523
524 return success();
525}
526
527//===----------------------------------------------------------------------===//
528// WarpgroupMmaOp
529//===----------------------------------------------------------------------===//
530
531LogicalResult isAllowedWGMMADataType(Type typeD, Type typeA, Type typeB) {
532 // F32 += F16 + F16
533 // F16 += F16 + F16
534 if (typeA.isF16() && typeB.isF16() && (typeD.isF32() || typeD.isF16()))
535 return success();
536 // F32 += TF32 + TF32
537 if (typeA.isTF32() && typeD.isF32() && typeB.isTF32())
538 return success();
539 // s32 += i8 + i8
540 if (typeA.isInteger(width: 8) && typeB.isInteger(width: 8) && typeD.isInteger(width: 32))
541 return success();
542 // s32 += i1 + i1
543 if (typeA.isInteger(width: 1) && typeB.isInteger(width: 1) && typeD.isInteger(width: 32))
544 return success();
545 // F32 += BF16 + BF16
546 // F16 += BF16 + BF16
547 if (typeA.isBF16() && typeB.isBF16() && (typeD.isF32() || typeD.isF16()))
548 return success();
549 // F16 += f8 + f8
550 // F32 += f8 + f8
551 if (isa<Float8E5M2Type, Float8E4M3FNType>(Val: typeA) &&
552 isa<Float8E5M2Type, Float8E4M3FNType>(Val: typeB) &&
553 (typeD.isF32() || typeD.isF16()))
554 return success();
555
556 return failure();
557}
558
559LogicalResult isAllowedSizeM(int sizeM) {
560 if (sizeM % kWgmmaSizeM)
561 return failure();
562 return success();
563}
564
565LogicalResult isAllowedSizeN(int sizeN, Type typeA) {
566 SmallVector<int> allowedN = {8, 16, 24, 32, 40, 48, 56, 64,
567 72, 80, 88, 96, 104, 112, 120, 128,
568 136, 144, 152, 160, 168, 176, 184, 192,
569 200, 208, 216, 224, 232, 240, 248, 256};
570 SmallVector<int> allowedNshort = {8, 16, 24, 32, 48, 64,
571 80, 96, 112, 128, 144, 160,
572 176, 192, 208, 224, 240, 256};
573 if (typeA.isBF16() || typeA.isF16() || typeA.isF32() || typeA.isTF32() ||
574 isa<Float8E5M2Type, Float8E4M3FNType>(Val: typeA))
575 if (llvm::is_contained(Range&: allowedN, Element: sizeN))
576 return success();
577
578 if (typeA.isInteger(width: 8) || typeA.isInteger(width: 1))
579 if (llvm::is_contained(Range&: allowedNshort, Element: sizeN))
580 return success();
581 return failure();
582}
583
584LogicalResult WarpgroupMmaOp::verify() {
585 if (getTransposeA() && !getTransposeB())
586 return emitOpError()
587 << "supports non-transpose A (Row Major) "
588 "and transpose B (Column Major) for the time being ";
589 MemRefType matrixA = getDescriptorA().getType().getTensor();
590 MemRefType matrixB = getDescriptorB().getType().getTensor();
591 VectorType matrixC = getMatrixC().getType().getFragmented();
592 VectorType matrixD = getMatrixD().getType().getFragmented();
593
594 if (matrixC != matrixD)
595 return emitOpError() << "type of matrix C and matrix D must be the same";
596
597 if (matrixA.getRank() != 2 || matrixB.getRank() != 2 ||
598 matrixC.getRank() != 2 || matrixD.getRank() != 2) {
599 return emitOpError()
600 << "has matrices A, B, C and D, they must be 2 dimensional";
601 }
602
603 if (matrixA.getShape()[1] != matrixB.getShape()[0])
604 return emitOpError() << "2nd dim matrix-A (" << matrixA.getShape()[1]
605 << ")!= 1st dim matrix-B (" << matrixB.getShape()[0]
606 << " )";
607 if (matrixA.getShape()[0] != matrixC.getShape()[0])
608 return emitOpError() << "1st dim matrix-A ( " << matrixA.getShape()[0]
609 << " )!= 1st dim matrix-C ( " << matrixC.getShape()[0]
610 << " )";
611 if (matrixB.getShape()[1] != matrixC.getShape()[1])
612 return emitOpError() << "2nd dim matrix-B ( " << matrixB.getShape()[1]
613 << " ) != 2nd dim matrix-C ( " << matrixC.getShape()[1]
614 << " )";
615
616 if (failed(Result: isAllowedWGMMADataType(typeD: matrixC.getElementType(),
617 typeA: matrixA.getElementType(),
618 typeB: matrixB.getElementType())))
619 return emitOpError() << matrixC.getElementType()
620 << " += " << matrixA.getElementType() << " * "
621 << matrixB.getElementType()
622 << ", it is not supported.";
623 // Check N
624 if (failed(Result: isAllowedSizeN(sizeN: matrixB.getDimSize(idx: 1), typeA: matrixA.getElementType()))) {
625 return emitOpError() << "has input type " << matrixB << " n is set to "
626 << matrixB.getDimSize(idx: 1) << ", it is not supported";
627 }
628
629 // Currently, f16/bf16 supported
630 if (!matrixC.getElementType().isF32() && !matrixA.getElementType().isF16() &&
631 !matrixA.getElementType().isBF16()) {
632 return emitOpError() << "hit a limitation: " << matrixC.getElementType()
633 << " += " << matrixA.getElementType() << " * "
634 << matrixB.getElementType()
635 << ", it is not supported yet";
636 }
637
638 return success();
639}
640
641LogicalResult WarpgroupMmaStoreOp::verify() {
642 MemRefType dstMemrefType = getDstMemref().getType();
643 VectorType vtype = getMatrixD().getType().getFragmented();
644
645 // Limitation
646 if (!vtype.getElementType().isF32()) {
647 return emitOpError()
648 << "hit a limitation: only f32 results for the time being";
649 }
650 if (vtype.getDimSize(idx: 0) != dstMemrefType.getDimSize(idx: 0) ||
651 vtype.getDimSize(idx: 1) != dstMemrefType.getDimSize(idx: 1)) {
652 return emitOpError() << "results [" << vtype << "][" << vtype.getDimSize(idx: 1)
653 << "] values. However, destination memref["
654 << dstMemrefType.getDimSize(idx: 0) << "]["
655 << dstMemrefType.getDimSize(idx: 1)
656 << "] does not have same size as results";
657 }
658 return success();
659}
660
661//===----------------------------------------------------------------------===//
662// WarpgroupMmaInitAccumulatorOp
663//===----------------------------------------------------------------------===//
664
665LogicalResult WarpgroupMmaInitAccumulatorOp::verify() {
666 WarpgroupAccumulatorType accType = getMatrixC().getType();
667 int64_t sizeM = accType.getFragmented().getDimSize(idx: 0);
668 int64_t sizeN = accType.getFragmented().getDimSize(idx: 1);
669 Type elemType = accType.getFragmented().getElementType();
670
671 if (failed(Result: isAllowedSizeM(sizeM)) ||
672 failed(Result: isAllowedSizeN(sizeN, typeA: elemType))) {
673 return emitOpError() << "has type " << accType.getFragmented()
674 << ". It does not fit into warp-group "
675 "level (wgmma) matrix multiplication instruction "
676 "(or not supported yet)";
677 }
678 return success();
679}
680
681//===----------------------------------------------------------------------===//
682// RcpOp
683//===----------------------------------------------------------------------===//
684
685LogicalResult RcpOp::verify() {
686 bool ftz = getFtz();
687 bool approx = getApprox();
688 mlir::NVVM::FPRoundingModeAttr rnd = getRoundingAttr();
689 // Currently, only `rcp_approx` and `ftz` is supported.
690 if (!approx || !ftz) {
691 return emitOpError()
692 << "has a limitation. non-approx or non-ftz is not supported yet.";
693 }
694 if (rnd.getValue() != mlir::NVVM::FPRoundingMode::NONE) {
695 return emitOpError() << "has a limitation. " << rnd
696 << " is not supported yet.";
697 }
698 return success();
699}
700
701//===----------------------------------------------------------------------===//
702// NVGPU_TruncfOp
703//===----------------------------------------------------------------------===//
704
705static LogicalResult verifyConversionShapes(Operation *op, Type inType,
706 Type outType) {
707 bool srcIsVector = llvm::isa<VectorType>(Val: inType);
708 bool dstIsVector = llvm::isa<VectorType>(Val: outType);
709 if (srcIsVector != dstIsVector)
710 return op->emitOpError(message: "input and output must both be scalars or both be "
711 "vectors, got ")
712 << inType << " and " << outType;
713 if (srcIsVector) {
714 auto srcVector = llvm::cast<VectorType>(Val&: inType);
715 auto dstVector = llvm::cast<VectorType>(Val&: outType);
716 if (srcVector.getShape() != dstVector.getShape())
717 return op->emitOpError(message: "input and output shapes must match, got ")
718 << inType << " and " << outType;
719 }
720 return success();
721}
722
723LogicalResult TruncfOp::verify() {
724 Type inType = getIn().getType();
725 Type outType = getType();
726 Type srcType = getElementTypeOrSelf(type: inType);
727 Type dstType = getElementTypeOrSelf(type: outType);
728 int srcBitWidth = srcType.getIntOrFloatBitWidth();
729 int dstBitWidth = dstType.getIntOrFloatBitWidth();
730 auto rnd = getRnd();
731
732 if (auto result = verifyConversionShapes(op: getOperation(), inType, outType);
733 failed(Result: result))
734 return result;
735
736 if (srcBitWidth <= dstBitWidth)
737 return emitOpError(message: "result type ")
738 << dstType << " must be narrower than operand type " << srcType;
739
740 if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
741 return emitOpError(message: "input type must be 64/32/16 bitwidth, but got ")
742 << srcBitWidth;
743
744 if (llvm::isa<Float8E8M0FNUType>(Val: dstType)) {
745 if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
746 rnd != mlir::NVVM::FPRoundingMode::RP)
747 return emitOpError(message: "expects RZ or RP rounding mode when result type is "
748 "e8m0, but got ")
749 << getRndAttr();
750 } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
751 // TODO: Currently, we only support conversions which fit into a single i32
752 // register. Support f32->f8/f6/f4 conversions with RS rounding.
753 if (!(srcBitWidth == 32 && dstBitWidth == 16))
754 return emitOpError(message: "RS (stochastic) rounding is only supported for "
755 "f32->f16/bf16, got ")
756 << srcType << " -> " << dstType;
757 if (!getRandomBits())
758 return emitOpError(message: "random_bits operand is required with RS rounding");
759 } else if (srcType.isF64() && dstBitWidth >= 16) {
760 if (rnd != mlir::NVVM::FPRoundingMode::RN)
761 return emitOpError(message: "expects RN rounding mode for f64 input, but got ")
762 << getRndAttr();
763 } else if (srcBitWidth == 32 && dstBitWidth == 16) {
764 if (rnd != mlir::NVVM::FPRoundingMode::RN &&
765 rnd != mlir::NVVM::FPRoundingMode::RZ)
766 return emitOpError(message: "expects RN or RZ rounding mode for f32 to f16/bf16, "
767 "but got ")
768 << getRndAttr();
769 } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
770 return emitOpError(message: "expects RN rounding mode, but got ") << getRndAttr();
771 }
772
773 if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
774 return emitOpError(message: "random_bits can only be used with RS rounding mode");
775
776 return success();
777}
778
779//===----------------------------------------------------------------------===//
780// NVGPU_ExtfOp
781//===----------------------------------------------------------------------===//
782
783LogicalResult ExtfOp::verify() {
784 Type inType = getIn().getType();
785 Type outType = getType();
786 Type srcType = getElementTypeOrSelf(type: inType);
787 Type dstType = getElementTypeOrSelf(type: outType);
788 int srcBitWidth = srcType.getIntOrFloatBitWidth();
789 int dstBitWidth = dstType.getIntOrFloatBitWidth();
790 auto rnd = getRnd();
791
792 if (failed(Result: verifyConversionShapes(op: getOperation(), inType, outType)))
793 return failure();
794
795 if (srcBitWidth >= dstBitWidth)
796 return emitOpError(message: "result type ")
797 << dstType << " must be wider than operand type " << srcType;
798
799 if (dstBitWidth != 16 && dstBitWidth != 32 && dstBitWidth != 64)
800 return emitOpError(message: "result type must be 16, 32, or 64 bitwidth, but got ")
801 << dstBitWidth;
802
803 if (llvm::isa<Float8E8M0FNUType>(Val: srcType) &&
804 !llvm::isa<BFloat16Type>(Val: dstType) && !dstType.isF32())
805 return emitOpError(message: "expects bf16 or f32 output type when input type is "
806 "e8m0.");
807
808 if (rnd != mlir::NVVM::FPRoundingMode::RN)
809 return emitOpError(message: "expects RN rounding mode, but got ") << getRndAttr();
810
811 if (getRelu() && llvm::isa<BFloat16Type>(Val: dstType))
812 return emitOpError(message: "relu is not supported for bf16 destination");
813
814 return success();
815}
816
817//===----------------------------------------------------------------------===//
818// TableGen'd dialect, type, and op definitions
819//===----------------------------------------------------------------------===//
820
821#define GET_ATTRDEF_CLASSES
822#include "mlir/Dialect/NVGPU/IR/NVGPUAttrDefs.cpp.inc"
823
824#include "mlir/Dialect/NVGPU/IR/NVGPUEnums.cpp.inc"
825
826#define GET_OP_CLASSES
827#include "mlir/Dialect/NVGPU/IR/NVGPUOps.cpp.inc"
828
829#define GET_TYPEDEF_CLASSES
830#include "mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.cpp.inc"
831