1//===- NVGPUToNVVM.cpp - NVGPU to NVVM dialect conversion -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h"
10
11#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
12#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
13#include "mlir/Conversion/LLVMCommon/Pattern.h"
14#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/GPU/IR/GPUDialect.h"
17#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
24#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
25#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
26#include "mlir/Dialect/MemRef/IR/MemRef.h"
27#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
28#include "mlir/Dialect/SCF/Transforms/Patterns.h"
29#include "mlir/Dialect/Vector/IR/VectorOps.h"
30#include "mlir/IR/BuiltinTypes.h"
31#include "mlir/IR/PatternMatch.h"
32#include "mlir/IR/TypeUtilities.h"
33#include "mlir/IR/Value.h"
34#include "mlir/Pass/Pass.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/DebugLog.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/raw_ostream.h"
39#include <optional>
40
41#define DEBUG_TYPE "nvgpu-to-nvvm"
42
43namespace mlir {
44#define GEN_PASS_DEF_CONVERTNVGPUTONVVMPASS
45#include "mlir/Conversion/Passes.h.inc"
46} // namespace mlir
47
48using namespace mlir;
49
50/// Number of bits that needs to be excluded when building matrix descriptor for
51/// wgmma operations.
52constexpr int exclude4LSB = 4;
53
54/// GPU has 32 bit registers, this function truncates values when larger width
55/// is not needed.
56static Value truncToI32(ImplicitLocOpBuilder &b, Value value) {
57 Type type = value.getType();
58 assert(llvm::isa<IntegerType>(type) && "expected an integer Value");
59 if (type.getIntOrFloatBitWidth() <= 32)
60 return value;
61 return LLVM::TruncOp::create(builder&: b, res: b.getI32Type(), arg: value);
62}
63
64/// Returns the type for the intrinsic given the vectorResultType of the
65/// `gpu.mma.sync` operation.
66static Type inferIntrinsicResultType(Type vectorResultType) {
67 MLIRContext *ctx = vectorResultType.getContext();
68 auto a = cast<LLVM::LLVMArrayType>(Val&: vectorResultType);
69 auto f16x2Ty = VectorType::get(shape: 2, elementType: Float16Type::get(context: ctx));
70 auto i32Ty = IntegerType::get(context: ctx, width: 32);
71 auto i32x2Ty = VectorType::get(shape: 2, elementType: i32Ty);
72 Type f64Ty = Float64Type::get(context: ctx);
73 Type f64x2Ty = VectorType::get(shape: 2, elementType: f64Ty);
74 Type f32Ty = Float32Type::get(context: ctx);
75 Type f32x2Ty = VectorType::get(shape: 2, elementType: f32Ty);
76 if (a.getElementType() == f16x2Ty) {
77 return LLVM::LLVMStructType::getLiteral(
78 context: ctx, types: SmallVector<Type>(a.getNumElements(), f16x2Ty));
79 }
80 if (a.getElementType() == i32x2Ty) {
81 return LLVM::LLVMStructType::getLiteral(
82 context: ctx,
83 types: SmallVector<Type>(static_cast<size_t>(a.getNumElements()) * 2, i32Ty));
84 }
85 if (a.getElementType() == f64x2Ty) {
86 return LLVM::LLVMStructType::getLiteral(context: ctx, types: {f64Ty, f64Ty});
87 }
88 if (a.getElementType() == f32x2Ty) {
89 return LLVM::LLVMStructType::getLiteral(
90 context: ctx,
91 types: SmallVector<Type>(static_cast<size_t>(a.getNumElements()) * 2, f32Ty));
92 }
93 if (a.getElementType() == VectorType::get(shape: 1, elementType: f32Ty)) {
94 return LLVM::LLVMStructType::getLiteral(
95 context: ctx, types: SmallVector<Type>(static_cast<size_t>(a.getNumElements()), f32Ty));
96 }
97 return vectorResultType;
98}
99
100/// Convert the SSA result of the NVVM intrinsic `nvvm.mma.sync` (which is
101/// always an LLVM struct) into a fragment that is compatible with the vector
102/// type of this operation. This involves extracting elements from the struct
103/// and inserting them into an LLVM array. These extra data-movement
104/// operations should be canonicalized away by the LLVM backend.
105static Value convertIntrinsicResult(Location loc, Type intrinsicResultType,
106 Type resultType, Value intrinsicResult,
107 RewriterBase &rewriter) {
108 MLIRContext *ctx = rewriter.getContext();
109 auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: intrinsicResultType);
110 auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: resultType);
111 Type i32Ty = rewriter.getI32Type();
112 Type f32Ty = rewriter.getF32Type();
113 Type f64Ty = rewriter.getF64Type();
114 Type f16x2Ty = VectorType::get(shape: 2, elementType: rewriter.getF16Type());
115 Type i32x2Ty = VectorType::get(shape: 2, elementType: i32Ty);
116 Type f64x2Ty = VectorType::get(shape: 2, elementType: f64Ty);
117 Type f32x2Ty = VectorType::get(shape: 2, elementType: f32Ty);
118 Type f32x1Ty = VectorType::get(shape: 1, elementType: f32Ty);
119
120 auto makeConst = [&](int32_t index) -> Value {
121 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: IntegerType::get(context: ctx, width: 32),
122 value: rewriter.getI32IntegerAttr(value: index));
123 };
124
125 if (arrayType) {
126 SmallVector<Value, 4> elements;
127
128 // The intrinsic returns 32-bit wide elements in a form which can be
129 // directly bitcasted and inserted into the result vector.
130 if (arrayType.getElementType() == f16x2Ty ||
131 arrayType.getElementType() == f32x1Ty) {
132 for (unsigned i = 0; i < structType.getBody().size(); i++) {
133 Value el =
134 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrinsicResult, position: i);
135 el = rewriter.createOrFold<LLVM::BitcastOp>(
136 location: loc, args: arrayType.getElementType(), args&: el);
137 elements.push_back(Elt: el);
138 }
139 }
140
141 // The intrinsic returns i32, f64, and f32 values as individual scalars,
142 // even when the result is notionally a 64-bit wide element (e.g. f32x2). We
143 // need to extract them from the struct and pack them into the 64-bit wide
144 // rows of the vector result.
145 if (arrayType.getElementType() == i32x2Ty ||
146 arrayType.getElementType() == f64x2Ty ||
147 arrayType.getElementType() == f32x2Ty) {
148
149 for (unsigned i = 0, e = structType.getBody().size() / 2; i < e; i++) {
150 Value vec =
151 LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: arrayType.getElementType());
152 Value x1 =
153 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrinsicResult, position: i * 2);
154 Value x2 = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrinsicResult,
155 position: i * 2 + 1);
156 vec = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, res: vec.getType(), vector: vec,
157 value: x1, position: makeConst(0));
158 vec = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, res: vec.getType(), vector: vec,
159 value: x2, position: makeConst(1));
160 elements.push_back(Elt: vec);
161 }
162 }
163
164 // Create the final vectorized result.
165 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: arrayType);
166 for (const auto &el : llvm::enumerate(First&: elements)) {
167 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: el.value(),
168 position: el.index());
169 }
170 return result;
171 }
172
173 return intrinsicResult;
174}
175
176/// The `gpu.mma.sync` converter below expects matrix fragment operands to be
177/// given as 2D `vectors` where the rows are 32b or 64b wide. The
178/// `nvvm.mma.sync` op expects these argments to be a given in a long list of
179/// scalars of certain types. This function helps unpack the `vector` arguments
180/// and cast them to the types expected by `nvvm.mma.sync`.
181static SmallVector<Value> unpackOperandVector(ImplicitLocOpBuilder &b,
182 Value operand,
183 NVVM::MMATypes operandPtxType) {
184 SmallVector<Value> result;
185 Type i32Ty = b.getI32Type();
186 Type f64Ty = b.getF64Type();
187 Type f32Ty = b.getF32Type();
188 Type i64Ty = b.getI64Type();
189 Type bf16x2Ty = VectorType::get(shape: 2, elementType: b.getBF16Type());
190 Type i8x4Ty = VectorType::get(shape: 4, elementType: b.getI8Type());
191 Type i4x8Ty = VectorType::get(shape: 8, elementType: b.getIntegerType(width: 4));
192 Type f32x1Ty = VectorType::get(shape: 1, elementType: f32Ty);
193 auto arrayTy = cast<LLVM::LLVMArrayType>(Val: operand.getType());
194
195 for (unsigned i = 0, e = arrayTy.getNumElements(); i < e; ++i) {
196 Value toUse = LLVM::ExtractValueOp::create(builder&: b, container: operand, position: i);
197
198 // For 4xi8 vectors, the intrinsic expects these to be provided as i32
199 // scalar types.
200 if (arrayTy.getElementType() == i8x4Ty ||
201 arrayTy.getElementType() == i4x8Ty ||
202 (arrayTy.getElementType() == bf16x2Ty &&
203 operandPtxType == NVVM::MMATypes::bf16) ||
204 (arrayTy.getElementType() == f32x1Ty &&
205 operandPtxType == NVVM::MMATypes::tf32)) {
206 result.push_back(Elt: LLVM::BitcastOp::create(builder&: b, res: i32Ty, arg: toUse));
207 continue;
208 }
209
210 // For some element types (i32, f32, f64), we need to unpack the inner
211 // vector/array type as well because the intrinsic expects individual
212 // scalars to be provided.
213 VectorType innerArrayTy = dyn_cast<VectorType>(Val: arrayTy.getElementType());
214 if (innerArrayTy && (innerArrayTy.getElementType() == i32Ty ||
215 innerArrayTy.getElementType() == f64Ty ||
216 innerArrayTy.getElementType() == f32Ty)) {
217 for (unsigned idx = 0, innerSize = innerArrayTy.getNumElements();
218 idx < innerSize; idx++) {
219 result.push_back(Elt: LLVM::ExtractElementOp::create(
220 builder&: b, vector: toUse,
221 position: LLVM::ConstantOp::create(builder&: b, res: i64Ty, value: b.getI64IntegerAttr(value: idx))));
222 }
223 continue;
224 }
225 result.push_back(Elt: toUse);
226 }
227 return result;
228}
229
230/// Returns whether mbarrier object has shared memory address space.
231static bool isMbarrierShared(nvgpu::MBarrierGroupType barrierType) {
232 return (mlir::nvgpu::NVGPUDialect::isSharedMemoryAddressSpace(
233 type: barrierType.getMemorySpace()));
234}
235
236/// Returns the memory space attribute of the mbarrier object.
237Attribute nvgpu::getMbarrierMemorySpace(MLIRContext *context,
238 nvgpu::MBarrierGroupType barrierType) {
239 Attribute memorySpace = {};
240 if (isMbarrierShared(barrierType)) {
241 memorySpace =
242 IntegerAttr::get(type: IntegerType::get(context, width: 64),
243 value: nvgpu::NVGPUDialect::kSharedMemoryAddressSpace);
244 }
245 return memorySpace;
246}
247
248/// Returns memref type of the mbarrier object. The type is defined in the
249/// MBarrierGroupType.
250MemRefType nvgpu::getMBarrierMemrefType(MLIRContext *context,
251 nvgpu::MBarrierGroupType barrierType) {
252 Attribute memorySpace = nvgpu::getMbarrierMemorySpace(context, barrierType);
253 MemRefLayoutAttrInterface layout;
254 return MemRefType::get(shape: {barrierType.getNumBarriers()},
255 elementType: IntegerType::get(context, width: 64), layout, memorySpace);
256}
257
258namespace {
259
260struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
261 using ConvertOpToLLVMPattern<nvgpu::LdMatrixOp>::ConvertOpToLLVMPattern;
262
263 LogicalResult
264 matchAndRewrite(nvgpu::LdMatrixOp op, OpAdaptor adaptor,
265 ConversionPatternRewriter &rewriter) const override {
266 MLIRContext *ctx = getContext();
267 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
268
269 // The result type of ldmatrix will always be a struct of 32bit integer
270 // registers if more than one 32bit value is returned. Otherwise, the result
271 // is a single i32. The result type of the GPU operation is always a vector
272 // of shape (NumRegisters, VectorRegister) where VectorRegister is the
273 // vector type of the result and always 32 bits long. We bitcast the result
274 // of the NVVM::LdMatrix to this vector type.
275 auto vectorResultType = dyn_cast<VectorType>(Val: op->getResultTypes()[0]);
276 if (!vectorResultType) {
277 return failure();
278 }
279 Type innerVectorType = VectorType::get(shape: vectorResultType.getDimSize(idx: 1),
280 elementType: vectorResultType.getElementType());
281
282 int64_t num32BitRegs = vectorResultType.getDimSize(idx: 0);
283
284 Type ldMatrixResultType;
285 if (num32BitRegs > 1) {
286 ldMatrixResultType = LLVM::LLVMStructType::getLiteral(
287 context: ctx, types: SmallVector<Type>(num32BitRegs, rewriter.getI32Type()));
288 } else {
289 ldMatrixResultType = rewriter.getI32Type();
290 }
291
292 auto srcMemrefType = cast<MemRefType>(Val: op.getSrcMemref().getType());
293 Value srcPtr =
294 getStridedElementPtr(rewriter, loc: b.getLoc(), type: srcMemrefType,
295 memRefDesc: adaptor.getSrcMemref(), indices: adaptor.getIndices());
296 auto shape = NVVM::LdStMatrixShapeAttr::get(context: rewriter.getContext(), m: 8, n: 8);
297 Value ldMatrixResult = NVVM::LdMatrixOp::create(
298 builder&: b, res: ldMatrixResultType, ptr: srcPtr,
299 /*num=*/op.getNumTiles(),
300 /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
301 : NVVM::MMALayout::row,
302 /*shape=*/shape, /*eltType=*/NVVM::LdStMatrixEltType::B16);
303
304 // The ldmatrix operation returns either a single i32 value or a struct of
305 // i32 values. Here we unpack those values and cast them back to their
306 // actual vector type (still of width 32b) and repack them into a result
307 // struct.
308 Type finalResultType = typeConverter->convertType(t: vectorResultType);
309 Value result = LLVM::PoisonOp::create(builder&: b, res: finalResultType);
310 for (int64_t i = 0, e = vectorResultType.getDimSize(idx: 0); i < e; i++) {
311 Value i32Register =
312 num32BitRegs > 1 ? LLVM::ExtractValueOp::create(builder&: b, container: ldMatrixResult, position: i)
313 : ldMatrixResult;
314 Value casted = LLVM::BitcastOp::create(builder&: b, res: innerVectorType, arg: i32Register);
315 result = LLVM::InsertValueOp::create(builder&: b, container: result, value: casted, position: i);
316 }
317
318 rewriter.replaceOp(op, newValues: result);
319 return success();
320 }
321};
322
323/// Convert the given type into the corresponding PTX type (NVVM::MMATypes
324/// enum).
325static FailureOr<NVVM::MMATypes> getNvvmMmaType(Type t) {
326 Type elType = getElementTypeOrSelf(type: t);
327 if (elType.isInteger(width: 8))
328 return elType.isUnsignedInteger() ? NVVM::MMATypes::u8 : NVVM::MMATypes::s8;
329 if (elType.isInteger(width: 4))
330 return elType.isUnsignedInteger() ? NVVM::MMATypes::u4 : NVVM::MMATypes::s4;
331 if (elType.isF16())
332 return NVVM::MMATypes::f16;
333 if (elType.isBF16())
334 return NVVM::MMATypes::bf16;
335 if (elType.isF64())
336 return NVVM::MMATypes::f64;
337 if (elType.isF32())
338 return NVVM::MMATypes::tf32;
339 if (elType.isF8E4M3FN())
340 return NVVM::MMATypes::e4m3;
341 if (elType.isF8E5M2())
342 return NVVM::MMATypes::e5m2;
343 return failure();
344}
345
346struct MmaSyncOptoNVVM : public ConvertOpToLLVMPattern<nvgpu::MmaSyncOp> {
347 using ConvertOpToLLVMPattern<nvgpu::MmaSyncOp>::ConvertOpToLLVMPattern;
348
349 LogicalResult
350 matchAndRewrite(nvgpu::MmaSyncOp op, OpAdaptor adaptor,
351 ConversionPatternRewriter &rewriter) const override {
352 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
353 // Get the shapes of the MMAMatrix type being used. The shapes will
354 // choose which intrinsic this op will be lowered to.
355 VectorType aType = op.getMatrixA().getType();
356 VectorType bType = op.getMatrixA().getType();
357 VectorType cType = op.getMatrixC().getType();
358
359 std::array<int64_t, 3> gemmShape = op.getMmaShapeAsArray();
360
361 // Tensor Cores (mma.sync) on F32 works only with TensorFloat32 (TF32).
362 bool tf32Enabled = op.getTf32Enabled().value_or(u: false);
363 if (aType.getElementType().isF32() && !tf32Enabled)
364 return failure();
365
366 FailureOr<NVVM::MMATypes> ptxTypeA = getNvvmMmaType(t: aType);
367 if (failed(Result: ptxTypeA))
368 return op->emitOpError(message: "failed to deduce operand PTX types");
369 FailureOr<NVVM::MMATypes> ptxTypeB = getNvvmMmaType(t: bType);
370 if (failed(Result: ptxTypeB))
371 return op->emitOpError(message: "failed to deduce operand PTX types");
372 std::optional<NVVM::MMATypes> ptxTypeC =
373 NVVM::MmaOp::inferOperandMMAType(operandElType: cType.getElementType(),
374 /*isAccumulator=*/true);
375 if (!ptxTypeC)
376 return op->emitError(
377 message: "could not infer the PTX type for the accumulator/result");
378
379 // TODO: add an attribute to the op to customize this behavior.
380 std::optional<NVVM::MMAIntOverflow> overflow(std::nullopt);
381 if (isa<IntegerType>(Val: aType.getElementType()))
382 overflow = NVVM::MMAIntOverflow::satfinite;
383
384 SmallVector<Value> matA =
385 unpackOperandVector(b, operand: adaptor.getMatrixA(), operandPtxType: *ptxTypeA);
386 SmallVector<Value> matB =
387 unpackOperandVector(b, operand: adaptor.getMatrixB(), operandPtxType: *ptxTypeB);
388 SmallVector<Value> matC =
389 unpackOperandVector(b, operand: adaptor.getMatrixC(), operandPtxType: *ptxTypeC);
390
391 Type desiredRetTy = typeConverter->convertType(t: op->getResultTypes()[0]);
392 Type intrinsicResTy = inferIntrinsicResultType(
393 vectorResultType: typeConverter->convertType(t: op->getResultTypes()[0]));
394 Value intrinsicResult =
395 NVVM::MmaOp::create(builder&: b, resultType: intrinsicResTy, operandA: matA, operandB: matB, operandC: matC,
396 /*shape=*/gemmShape,
397 /*b1Op=*/std::nullopt,
398 /*intOverflow=*/overflow,
399 /*multiplicandPtxTypes=*/
400 std::array<NVVM::MMATypes, 2>{*ptxTypeA, *ptxTypeB},
401 /*multiplicandLayouts=*/
402 std::array<NVVM::MMALayout, 2>{
403 NVVM::MMALayout::row, NVVM::MMALayout::col});
404 rewriter.replaceOp(op, newValues: convertIntrinsicResult(loc: op.getLoc(), intrinsicResultType: intrinsicResTy,
405 resultType: desiredRetTy, intrinsicResult,
406 rewriter));
407 return success();
408 }
409};
410
411struct ConvertNVGPUToNVVMPass
412 : public impl::ConvertNVGPUToNVVMPassBase<ConvertNVGPUToNVVMPass> {
413 using Base::Base;
414
415 void runOnOperation() override {
416 LowerToLLVMOptions options(&getContext());
417 RewritePatternSet patterns(&getContext());
418 LLVMTypeConverter converter(&getContext(), options);
419 IRRewriter rewriter(&getContext());
420 nvgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter);
421
422 /// device-side async tokens cannot be materialized in nvvm. We just
423 /// convert them to a dummy i32 type in order to easily drop them during
424 /// conversion.
425 converter.addConversion(callback: [&](nvgpu::DeviceAsyncTokenType type) -> Type {
426 return converter.convertType(t: IntegerType::get(context: type.getContext(), width: 32));
427 });
428 converter.addConversion(callback: [&](nvgpu::WarpgroupAccumulatorType type) -> Type {
429 Type elemType = type.getFragmented().getElementType();
430 int64_t sizeM = type.getFragmented().getDimSize(idx: 0);
431 int64_t sizeN = type.getFragmented().getDimSize(idx: 1);
432
433 unsigned numMembers;
434 if (elemType.isF32() || elemType.isInteger(width: 32))
435 numMembers = sizeN / 2;
436 else if (elemType.isF16())
437 numMembers = sizeN / 4;
438 else
439 llvm_unreachable("unsupported type for warpgroup accumulator");
440
441 SmallVector<Type> innerStructBody;
442 for (unsigned i = 0; i < numMembers; i++)
443 innerStructBody.push_back(Elt: elemType);
444 auto innerStructType =
445 LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: innerStructBody);
446
447 SmallVector<Type> structBody;
448 for (int i = 0; i < sizeM; i += kWgmmaSizeM)
449 structBody.push_back(Elt: innerStructType);
450
451 auto convertedType =
452 LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: structBody);
453 return converter.convertType(t: convertedType);
454 });
455 converter.addConversion(callback: [&](nvgpu::MBarrierTokenType type) -> Type {
456 return converter.convertType(t: IntegerType::get(context: type.getContext(), width: 64));
457 });
458 converter.addConversion(
459 callback: [&](nvgpu::WarpgroupMatrixDescriptorType type) -> Type {
460 return converter.convertType(t: IntegerType::get(context: type.getContext(), width: 64));
461 });
462 converter.addConversion(callback: [&](nvgpu::MBarrierGroupType type) -> Type {
463 return converter.convertType(
464 t: nvgpu::getMBarrierMemrefType(context: rewriter.getContext(), barrierType: type));
465 });
466 converter.addConversion(callback: [&](nvgpu::TensorMapDescriptorType type) -> Type {
467 return LLVM::LLVMPointerType::get(context: type.getContext());
468 });
469 populateNVGPUToNVVMConversionPatterns(converter, patterns);
470 LLVMConversionTarget target(getContext());
471 target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
472 target.addLegalDialect<::mlir::arith::ArithDialect>();
473 target.addLegalDialect<::mlir::memref::MemRefDialect>();
474 target.addLegalDialect<::mlir::NVVM::NVVMDialect>();
475 target.addLegalDialect<::mlir::vector::VectorDialect>();
476 mlir::scf::populateSCFStructuralTypeConversionsAndLegality(
477 typeConverter: converter, patterns, target);
478 if (failed(Result: applyPartialConversion(op: getOperation(), target,
479 patterns: std::move(patterns))))
480 signalPassFailure();
481 }
482};
483
484/// Returns the constraints for the sparse MMA inline assembly instruction.
485static std::string buildMmaSparseAsmConstraintString(unsigned matASize,
486 unsigned matBSize,
487 unsigned matCSize) {
488 std::string str;
489 llvm::raw_string_ostream ss(str);
490 for (unsigned i = 0; i < matCSize; i++)
491 ss << "=r,";
492 for (unsigned i = 0; i < matASize + matBSize + matCSize; i++)
493 ss << "r,";
494 // The final operand is for the sparsity metadata.
495 // The sparsity selector appears as direct literal.
496 ss << "r";
497 return str;
498}
499
500/// Returns the string for the `mma.sp.sync` instruction that corresponds to
501/// the given parameters. Note that this function doesn't do any validation,
502/// it's expected that the provided parameters correspond to a valid
503/// instruction.
504static std::string buildMmaSparseAsmString(
505 const std::array<int64_t, 3> &shape, unsigned matASize, unsigned matBSize,
506 unsigned matCSize, NVVM::MMATypes ptxTypeA, NVVM::MMATypes ptxTypeB,
507 NVVM::MMATypes ptxTypeC, NVVM::MMATypes ptxTypeD,
508 std::optional<NVVM::MMAIntOverflow> overflow, unsigned metaDataSelector) {
509 auto ptxTypeStr = [](NVVM::MMATypes ptxType) {
510 return NVVM::stringifyMMATypes(ptxType);
511 };
512
513 std::string asmStr;
514 llvm::raw_string_ostream ss(asmStr);
515 ss << "mma.sp.sync.aligned.m" << shape[0] << "n" << shape[1] << "k"
516 << shape[2] << ".row.col.";
517
518 if (overflow)
519 ss << NVVM::stringifyMMAIntOverflow(*overflow) << ".";
520
521 ss << ptxTypeStr(ptxTypeD) << "." << ptxTypeStr(ptxTypeA) << "."
522 << ptxTypeStr(ptxTypeB) << "." << ptxTypeStr(ptxTypeC) << " ";
523 unsigned asmArgIdx = 0;
524
525 // The operand string is structured into sections `{matC elements...},
526 // {matA elements...}, {matB elements...}, {matC elements}`.
527 for (const auto arrSize : {matCSize, matASize, matBSize, matCSize}) {
528 ss << "{";
529 for (unsigned i = 0; i < arrSize; i++)
530 ss << "$" << asmArgIdx++ << (i < arrSize - 1 ? "," : "");
531 ss << "},";
532 }
533 ss << "$" << asmArgIdx++ << ",";
534 assert(metaDataSelector <= 1);
535 ss << "0x" << metaDataSelector << ";";
536 return asmStr;
537}
538
539/// Builds an inline assembly operation corresponding to the specified MMA
540/// sparse sync operation.
541static FailureOr<LLVM::InlineAsmOp> emitMmaSparseSyncOpAsm(
542 ImplicitLocOpBuilder &b, NVVM::MMATypes ptxTypeA, NVVM::MMATypes ptxTypeB,
543 NVVM::MMATypes ptxTypeC, NVVM::MMATypes ptxTypeD,
544 std::optional<NVVM::MMAIntOverflow> overflow, ArrayRef<Value> unpackedAData,
545 ArrayRef<Value> unpackedB, ArrayRef<Value> unpackedC, Value indexData,
546 int64_t metadataSelector, const std::array<int64_t, 3> &shape,
547 Type intrinsicResultType) {
548 auto asmDialectAttr =
549 LLVM::AsmDialectAttr::get(context: b.getContext(), val: LLVM::AsmDialect::AD_ATT);
550
551 const unsigned matASize = unpackedAData.size();
552 const unsigned matBSize = unpackedB.size();
553 const unsigned matCSize = unpackedC.size();
554
555 std::string asmStr = buildMmaSparseAsmString(
556 shape, matASize, matBSize, matCSize, ptxTypeA, ptxTypeB, ptxTypeC,
557 ptxTypeD, overflow, metaDataSelector: metadataSelector);
558 std::string constraintStr =
559 buildMmaSparseAsmConstraintString(matASize, matBSize, matCSize);
560
561 SmallVector<Value> asmVals;
562 asmVals.reserve(N: matASize + matBSize + matCSize + 1);
563 for (ArrayRef<Value> args : {unpackedAData, unpackedB, unpackedC})
564 llvm::append_range(C&: asmVals, R&: args);
565 asmVals.push_back(Elt: indexData);
566
567 return LLVM::InlineAsmOp::create(builder&: b,
568 /*resultTypes=*/res: intrinsicResultType,
569 /*operands=*/asmVals,
570 /*asm_string=*/asmStr,
571 /*constraints=*/constraintStr,
572 /*has_side_effects=*/true,
573 /*is_align_stack=*/false,
574 tail_call_kind: LLVM::TailCallKind::None,
575 /*convergent=*/false,
576 /*asm_dialect=*/asmDialectAttr,
577 /*operand_attrs=*/ArrayAttr());
578}
579
580/// Lowers `nvgpu.mma.sp.sync` to inline assembly.
581struct NVGPUMmaSparseSyncLowering
582 : public ConvertOpToLLVMPattern<nvgpu::MmaSparseSyncOp> {
583 using ConvertOpToLLVMPattern<nvgpu::MmaSparseSyncOp>::ConvertOpToLLVMPattern;
584
585 LogicalResult
586 matchAndRewrite(nvgpu::MmaSparseSyncOp op, OpAdaptor adaptor,
587 ConversionPatternRewriter &rewriter) const override {
588 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
589 // Get the shapes of the MMAMatrix type being used. The shapes will
590 // choose which intrinsic this op will be lowered to.
591 VectorType aType = op.getMatrixA().getType();
592 VectorType bType = op.getMatrixB().getType();
593 VectorType cType = op.getMatrixC().getType();
594
595 FailureOr<NVVM::MMATypes> ptxTypeA = getNvvmMmaType(t: aType);
596 if (failed(Result: ptxTypeA))
597 return op->emitOpError(message: "failed to deduce operand PTX types");
598 FailureOr<NVVM::MMATypes> ptxTypeB = getNvvmMmaType(t: bType);
599 if (failed(Result: ptxTypeB))
600 return op->emitOpError(message: "failed to deduce operand PTX types");
601 std::optional<NVVM::MMATypes> ptxTypeC =
602 NVVM::MmaOp::inferOperandMMAType(operandElType: cType.getElementType(),
603 /*isAccumulator=*/true);
604 if (!ptxTypeC)
605 return op->emitError(
606 message: "could not infer the PTX type for the accumulator/result");
607
608 // Same as `mma.sync`, F32 works only with TensorFloat32 (TF32).
609 bool tf32Enabled = op.getTf32Enabled().value_or(u: false);
610 if (aType.getElementType().isF32() && !tf32Enabled)
611 return failure();
612
613 // TODO: add an attribute to the op to customize this behavior.
614 std::optional<NVVM::MMAIntOverflow> overflow(std::nullopt);
615 if (isa<IntegerType>(Val: aType.getElementType()))
616 overflow = NVVM::MMAIntOverflow::satfinite;
617
618 SmallVector<Value> matA =
619 unpackOperandVector(b, operand: adaptor.getMatrixA(), operandPtxType: *ptxTypeA);
620 SmallVector<Value> matB =
621 unpackOperandVector(b, operand: adaptor.getMatrixB(), operandPtxType: *ptxTypeB);
622 SmallVector<Value> matC =
623 unpackOperandVector(b, operand: adaptor.getMatrixC(), operandPtxType: *ptxTypeC);
624
625 Type desiredRetTy = typeConverter->convertType(t: op->getResultTypes()[0]);
626 Type intrinsicResTy = inferIntrinsicResultType(
627 vectorResultType: typeConverter->convertType(t: op->getResultTypes()[0]));
628
629 // Bitcast the sparse metadata from vector<2xf16> to an i32.
630 Value sparseMetadata = adaptor.getSparseMetadata();
631 if (sparseMetadata.getType() != VectorType::get(shape: 2, elementType: rewriter.getI16Type()))
632 return op->emitOpError() << "Expected metadata type to be LLVM "
633 "VectorType of 2 i16 elements";
634 sparseMetadata =
635 LLVM::BitcastOp::create(builder&: b, res: rewriter.getI32Type(), arg: sparseMetadata);
636
637 FailureOr<LLVM::InlineAsmOp> intrinsicResult = emitMmaSparseSyncOpAsm(
638 b, ptxTypeA: *ptxTypeA, ptxTypeB: *ptxTypeB, ptxTypeC: *ptxTypeC, ptxTypeD: *ptxTypeC, overflow, unpackedAData: matA, unpackedB: matB,
639 unpackedC: matC, indexData: sparseMetadata, metadataSelector: op.getSparsitySelector(), shape: op.getMmaShapeAsArray(),
640 intrinsicResultType: intrinsicResTy);
641 if (failed(Result: intrinsicResult))
642 return failure();
643
644 assert((*intrinsicResult).getNumResults() == 1 &&
645 "expected inline asm op returns a single LLVM struct type");
646 rewriter.replaceOp(
647 op, newValues: convertIntrinsicResult(loc: op.getLoc(), intrinsicResultType: intrinsicResTy, resultType: desiredRetTy,
648 intrinsicResult: (*intrinsicResult)->getResult(idx: 0), rewriter));
649 return success();
650 }
651};
652
653struct NVGPUAsyncCopyLowering
654 : public ConvertOpToLLVMPattern<nvgpu::DeviceAsyncCopyOp> {
655 using ConvertOpToLLVMPattern<
656 nvgpu::DeviceAsyncCopyOp>::ConvertOpToLLVMPattern;
657
658 LogicalResult
659 matchAndRewrite(nvgpu::DeviceAsyncCopyOp op, OpAdaptor adaptor,
660 ConversionPatternRewriter &rewriter) const override {
661 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
662 Location loc = op.getLoc();
663 auto dstMemrefType = cast<MemRefType>(Val: op.getDst().getType());
664 Value dstPtr =
665 getStridedElementPtr(rewriter, loc: b.getLoc(), type: dstMemrefType,
666 memRefDesc: adaptor.getDst(), indices: adaptor.getDstIndices());
667 FailureOr<unsigned> dstAddressSpace =
668 getTypeConverter()->getMemRefAddressSpace(type: dstMemrefType);
669 if (failed(Result: dstAddressSpace))
670 return rewriter.notifyMatchFailure(
671 arg&: loc, msg: "destination memref address space not convertible to integer");
672
673 auto srcMemrefType = cast<MemRefType>(Val: op.getSrc().getType());
674 FailureOr<unsigned> srcAddressSpace =
675 getTypeConverter()->getMemRefAddressSpace(type: srcMemrefType);
676 if (failed(Result: srcAddressSpace))
677 return rewriter.notifyMatchFailure(
678 arg&: loc, msg: "source memref address space not convertible to integer");
679
680 Value scrPtr =
681 getStridedElementPtr(rewriter, loc, type: srcMemrefType, memRefDesc: adaptor.getSrc(),
682 indices: adaptor.getSrcIndices());
683 // Intrinsics takes a global pointer so we need an address space cast.
684 auto srcPointerGlobalType = LLVM::LLVMPointerType::get(
685 context: op->getContext(), addressSpace: static_cast<unsigned>(NVVM::NVVMMemorySpace::Global));
686 scrPtr = LLVM::AddrSpaceCastOp::create(builder&: b, res: srcPointerGlobalType, arg: scrPtr);
687 int64_t dstElements = adaptor.getDstElements().getZExtValue();
688 int64_t sizeInBytes =
689 (dstMemrefType.getElementTypeBitWidth() * dstElements) / 8;
690 // When the optional SrcElements argument is *not* present, the regular
691 // CpAsyncOp is generated. CopyAsyncOp reads bytes from source (global
692 // memory) to fill DstElements number of elements in the destination
693 // (shared memory).
694 Value srcBytes = adaptor.getSrcElements();
695 if (srcBytes) {
696 // When the optional SrcElements argument is present, the source (global
697 // memory) of CpAsyncOp is read only for SrcElements number of elements.
698 // The rest of the DstElements in the destination (shared memory) are
699 // filled with zeros.
700 Value c3I32 =
701 LLVM::ConstantOp::create(builder&: b, res: b.getI32Type(), value: b.getI32IntegerAttr(value: 3));
702 Value bitwidth = LLVM::ConstantOp::create(
703 builder&: b, res: b.getI32Type(),
704 value: b.getI32IntegerAttr(value: srcMemrefType.getElementTypeBitWidth()));
705 Value srcElementsI32 = LLVM::TruncOp::create(builder&: b, res: b.getI32Type(), arg: srcBytes);
706 srcBytes = LLVM::LShrOp::create(
707 builder&: b, lhs: LLVM::MulOp::create(builder&: b, lhs: bitwidth, rhs: srcElementsI32), rhs: c3I32);
708 }
709 // Cache global (.cg) for 16 dst bytes, Cache all (.ca) for sizes other than
710 // 16 dst bytes.
711 NVVM::LoadCacheModifierKind cacheModifier =
712 (op.getBypassL1().value_or(u: false) && sizeInBytes == 16)
713 ? NVVM::LoadCacheModifierKind::CG
714 : NVVM::LoadCacheModifierKind::CA;
715
716 NVVM::CpAsyncOp::create(
717 builder&: b, dst: dstPtr, src: scrPtr, size: rewriter.getI32IntegerAttr(value: sizeInBytes),
718 modifier: NVVM::LoadCacheModifierKindAttr::get(context: op->getContext(), value: cacheModifier),
719 cpSize: srcBytes);
720
721 // Drop the result token.
722 Value zero =
723 LLVM::ConstantOp::create(builder&: b, res: IntegerType::get(context: op.getContext(), width: 32),
724 value: rewriter.getI32IntegerAttr(value: 0));
725 rewriter.replaceOp(op, newValues: zero);
726 return success();
727 }
728};
729
730struct NVGPUAsyncCreateGroupLowering
731 : public ConvertOpToLLVMPattern<nvgpu::DeviceAsyncCreateGroupOp> {
732 using ConvertOpToLLVMPattern<
733 nvgpu::DeviceAsyncCreateGroupOp>::ConvertOpToLLVMPattern;
734
735 LogicalResult
736 matchAndRewrite(nvgpu::DeviceAsyncCreateGroupOp op, OpAdaptor adaptor,
737 ConversionPatternRewriter &rewriter) const override {
738 NVVM::CpAsyncCommitGroupOp::create(builder&: rewriter, location: op.getLoc());
739 // Drop the result token.
740 Value zero = LLVM::ConstantOp::create(builder&: rewriter, location: op->getLoc(),
741 res: IntegerType::get(context: op.getContext(), width: 32),
742 value: rewriter.getI32IntegerAttr(value: 0));
743 rewriter.replaceOp(op, newValues: zero);
744 return success();
745 }
746};
747
748struct NVGPUAsyncWaitLowering
749 : public ConvertOpToLLVMPattern<nvgpu::DeviceAsyncWaitOp> {
750 using ConvertOpToLLVMPattern<
751 nvgpu::DeviceAsyncWaitOp>::ConvertOpToLLVMPattern;
752
753 LogicalResult
754 matchAndRewrite(nvgpu::DeviceAsyncWaitOp op, OpAdaptor adaptor,
755 ConversionPatternRewriter &rewriter) const override {
756 // If numGroup is not present pick 0 as a conservative correct value.
757 int32_t numGroups = adaptor.getNumGroups().value_or(u: 0);
758 NVVM::CpAsyncWaitGroupOp::create(builder&: rewriter, location: op.getLoc(), n: numGroups);
759 rewriter.eraseOp(op);
760 return success();
761 }
762};
763
764/// Creates mbarrier object in shared memory
765struct NVGPUMBarrierCreateLowering
766 : public ConvertOpToLLVMPattern<nvgpu::MBarrierCreateOp> {
767 using ConvertOpToLLVMPattern<nvgpu::MBarrierCreateOp>::ConvertOpToLLVMPattern;
768
769 template <typename moduleT>
770 memref::GlobalOp generateGlobalBarrier(ConversionPatternRewriter &rewriter,
771 Operation *funcOp, moduleT moduleOp,
772 MemRefType barrierType) const {
773 SymbolTable symbolTable(moduleOp);
774 OpBuilder::InsertionGuard guard(rewriter);
775 rewriter.setInsertionPoint(&moduleOp.front());
776 auto global = memref::GlobalOp::create(
777 builder&: rewriter, location: funcOp->getLoc(), sym_name: "__mbarrier",
778 /*sym_visibility=*/rewriter.getStringAttr(bytes: "private"),
779 /*type=*/barrierType,
780 /*initial_value=*/ElementsAttr(),
781 /*constant=*/false,
782 /*alignment=*/rewriter.getI64IntegerAttr(value: 8));
783 symbolTable.insert(symbol: global);
784 return global;
785 }
786
787 LogicalResult
788 matchAndRewrite(nvgpu::MBarrierCreateOp op, OpAdaptor adaptor,
789 ConversionPatternRewriter &rewriter) const override {
790 Operation *funcOp = op->getParentOp();
791 MemRefType barrierType = nvgpu::getMBarrierMemrefType(
792 context: rewriter.getContext(), barrierType: op.getBarriers().getType());
793
794 memref::GlobalOp global;
795 if (auto moduleOp = funcOp->getParentOfType<gpu::GPUModuleOp>())
796 global = generateGlobalBarrier(rewriter, funcOp, moduleOp, barrierType);
797 else if (auto moduleOp = funcOp->getParentOfType<ModuleOp>())
798 global = generateGlobalBarrier(rewriter, funcOp, moduleOp, barrierType);
799
800 rewriter.setInsertionPoint(op);
801 rewriter.replaceOpWithNewOp<memref::GetGlobalOp>(op, args&: barrierType,
802 args: global.getName());
803 return success();
804 }
805};
806
807/// Base class for lowering mbarrier operations to nvvm intrinsics.
808template <typename SourceOp>
809struct MBarrierBasePattern : public ConvertOpToLLVMPattern<SourceOp> {
810public:
811 using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
812 /// Returns the base pointer of the mbarrier object.
813 Value getMbarrierPtr(ImplicitLocOpBuilder &b,
814 nvgpu::MBarrierGroupType mbarType, Value memrefDesc,
815 Value mbarId,
816 ConversionPatternRewriter &rewriter) const {
817 MemRefType mbarrierMemrefType =
818 nvgpu::getMBarrierMemrefType(context: rewriter.getContext(), barrierType: mbarType);
819 return ConvertToLLVMPattern::getStridedElementPtr(
820 rewriter, loc: b.getLoc(), type: mbarrierMemrefType, memRefDesc: memrefDesc, indices: {mbarId});
821 }
822};
823
824struct NVGPUMBarrierGetLowering
825 : public MBarrierBasePattern<nvgpu::MBarrierGetOp> {
826 using MBarrierBasePattern<nvgpu::MBarrierGetOp>::MBarrierBasePattern;
827
828 LogicalResult
829 matchAndRewrite(nvgpu::MBarrierGetOp op, OpAdaptor adaptor,
830 ConversionPatternRewriter &rewriter) const override {
831 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
832 nvgpu::MBarrierGroupType mbarrierType = op.getBarriers().getType();
833 rewriter.setInsertionPoint(op);
834 Value barrier = getMbarrierPtr(b, mbarType: mbarrierType, memrefDesc: adaptor.getBarriers(),
835 mbarId: adaptor.getMbarId(), rewriter);
836 Type resType = op.getMbarrierPointer().getType();
837 rewriter.replaceOpWithNewOp<LLVM::PtrToIntOp>(op, args&: resType, args&: barrier);
838 return success();
839 }
840};
841
842/// Lowers `nvgpu.mbarrier.init` to `nvvm.mbarrier.init`
843struct NVGPUMBarrierInitLowering
844 : public MBarrierBasePattern<nvgpu::MBarrierInitOp> {
845 using MBarrierBasePattern<nvgpu::MBarrierInitOp>::MBarrierBasePattern;
846
847 LogicalResult
848 matchAndRewrite(nvgpu::MBarrierInitOp op, OpAdaptor adaptor,
849 ConversionPatternRewriter &rewriter) const override {
850 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
851 nvgpu::MBarrierGroupType mbarrierType = op.getBarriers().getType();
852 rewriter.setInsertionPoint(op);
853 Value barrier = getMbarrierPtr(b, mbarType: mbarrierType, memrefDesc: adaptor.getBarriers(),
854 mbarId: adaptor.getMbarId(), rewriter);
855 Value count = truncToI32(b, value: adaptor.getCount());
856 rewriter.replaceOpWithNewOp<NVVM::MBarrierInitOp>(op, args&: barrier, args&: count, args: 0,
857 args: adaptor.getPredicate());
858 return success();
859 }
860};
861
862/// Lowers `nvgpu.mbarrier.arrive` to `nvvm.mbarrier.arrive`
863struct NVGPUMBarrierArriveLowering
864 : public MBarrierBasePattern<nvgpu::MBarrierArriveOp> {
865 using MBarrierBasePattern<nvgpu::MBarrierArriveOp>::MBarrierBasePattern;
866 LogicalResult
867 matchAndRewrite(nvgpu::MBarrierArriveOp op, OpAdaptor adaptor,
868 ConversionPatternRewriter &rewriter) const override {
869 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
870 Value barrier =
871 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
872 mbarId: adaptor.getMbarId(), rewriter);
873 rewriter.replaceOpWithNewOp<NVVM::MBarrierArriveOp>(
874 op, args&: barrier, /*count=*/args: Value{}, /*multicastMask=*/args: Value{});
875 return success();
876 }
877};
878
879/// Lowers `nvgpu.mbarrier.arrive.nocomplete` to
880/// `nvvm.mbarrier.arrive.nocomplete`
881struct NVGPUMBarrierArriveNoCompleteLowering
882 : public MBarrierBasePattern<nvgpu::MBarrierArriveNoCompleteOp> {
883 using MBarrierBasePattern<
884 nvgpu::MBarrierArriveNoCompleteOp>::MBarrierBasePattern;
885 LogicalResult
886 matchAndRewrite(nvgpu::MBarrierArriveNoCompleteOp op, OpAdaptor adaptor,
887 ConversionPatternRewriter &rewriter) const override {
888 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
889 Value barrier =
890 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
891 mbarId: adaptor.getMbarId(), rewriter);
892 Type tokenType = getTypeConverter()->convertType(
893 t: nvgpu::MBarrierTokenType::get(ctx: op->getContext()));
894 Value count = truncToI32(b, value: adaptor.getCount());
895 rewriter.replaceOpWithNewOp<NVVM::MBarrierArriveNocompleteOp>(
896 op, args&: tokenType, args&: barrier, args&: count);
897 return success();
898 }
899};
900
901/// Lowers `nvgpu.mbarrier.test.wait` to `nvvm.mbarrier.test.wait`
902struct NVGPUMBarrierTestWaitLowering
903 : public MBarrierBasePattern<nvgpu::MBarrierTestWaitOp> {
904 using MBarrierBasePattern<nvgpu::MBarrierTestWaitOp>::MBarrierBasePattern;
905 LogicalResult
906 matchAndRewrite(nvgpu::MBarrierTestWaitOp op, OpAdaptor adaptor,
907 ConversionPatternRewriter &rewriter) const override {
908 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
909 Value barrier =
910 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
911 mbarId: adaptor.getMbarId(), rewriter);
912 Type retType = rewriter.getI1Type();
913 rewriter.replaceOpWithNewOp<NVVM::MBarrierTestWaitOp>(op, args&: retType, args&: barrier,
914 args: adaptor.getToken());
915 return success();
916 }
917};
918
919struct NVGPUMBarrierArriveExpectTxLowering
920 : public MBarrierBasePattern<nvgpu::MBarrierArriveExpectTxOp> {
921 using MBarrierBasePattern<
922 nvgpu::MBarrierArriveExpectTxOp>::MBarrierBasePattern;
923 LogicalResult
924 matchAndRewrite(nvgpu::MBarrierArriveExpectTxOp op, OpAdaptor adaptor,
925 ConversionPatternRewriter &rewriter) const override {
926 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
927 Value barrier =
928 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
929 mbarId: adaptor.getMbarId(), rewriter);
930 Value txcount = truncToI32(b, value: adaptor.getTxcount());
931 NVVM::MBarrierArriveExpectTxOp::create(
932 builder&: rewriter, location: op->getLoc(), addr: barrier, txcount, /*multicastMask=*/Value{},
933 scope: NVVM::MemScopeKind::CTA, /*relaxed=*/false, predicate: adaptor.getPredicate());
934 rewriter.eraseOp(op);
935 return success();
936 }
937};
938
939struct NVGPUMBarrierTryWaitParityLowering
940 : public MBarrierBasePattern<nvgpu::MBarrierTryWaitParityOp> {
941 using MBarrierBasePattern<
942 nvgpu::MBarrierTryWaitParityOp>::MBarrierBasePattern;
943 LogicalResult
944 matchAndRewrite(nvgpu::MBarrierTryWaitParityOp op, OpAdaptor adaptor,
945 ConversionPatternRewriter &rewriter) const override {
946 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
947 Value barrier =
948 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
949 mbarId: adaptor.getMbarId(), rewriter);
950 Value ticks = truncToI32(b, value: adaptor.getTicks());
951 Value phase =
952 LLVM::ZExtOp::create(builder&: b, res: b.getI32Type(), arg: adaptor.getPhaseParity());
953 rewriter.replaceOpWithNewOp<NVVM::MBarrierTryWaitParityOp>(op, args&: barrier,
954 args&: phase, args&: ticks);
955 return success();
956 }
957};
958
959struct NVGPUTmaAsyncLoadOpLowering
960 : public MBarrierBasePattern<nvgpu::TmaAsyncLoadOp> {
961 using MBarrierBasePattern<nvgpu::TmaAsyncLoadOp>::MBarrierBasePattern;
962 LogicalResult
963 matchAndRewrite(nvgpu::TmaAsyncLoadOp op, OpAdaptor adaptor,
964 ConversionPatternRewriter &rewriter) const override {
965 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
966 auto srcMemrefType = cast<MemRefType>(Val: op.getDst().getType());
967 Value dest = getStridedElementPtr(rewriter, loc: op->getLoc(), type: srcMemrefType,
968 memRefDesc: adaptor.getDst(), indices: {});
969 // Intrinsics takes a shared-cluster pointer so we need an
970 // address space cast from 3 to 7.
971 // TODO: Introduce AS(7) in NVGPU.
972 auto ptrSharedClusterType = LLVM::LLVMPointerType::get(
973 context: op->getContext(),
974 addressSpace: static_cast<unsigned>(NVVM::NVVMMemorySpace::SharedCluster));
975 dest = LLVM::AddrSpaceCastOp::create(builder&: b, res: ptrSharedClusterType, arg: dest);
976
977 Value barrier =
978 getMbarrierPtr(b, mbarType: op.getBarriers().getType(), memrefDesc: adaptor.getBarriers(),
979 mbarId: adaptor.getMbarId(), rewriter);
980
981 SmallVector<Value> coords = adaptor.getCoordinates();
982 for (auto [index, value] : llvm::enumerate(First&: coords)) {
983 coords[index] = truncToI32(b, value);
984 }
985
986 // TODO: Enhance the NVGPU Op for other modes too
987 rewriter.replaceOpWithNewOp<NVVM::CpAsyncBulkTensorGlobalToSharedClusterOp>(
988 op, args&: dest, args: adaptor.getTensorMapDescriptor(), args&: coords, args&: barrier,
989 args: ValueRange{}, args: adaptor.getMulticastMask(), args: Value{},
990 args: NVVM::TMALoadMode::TILE, // default is TILE mode
991 args: false, // default is cluster-scope
992 args: nullptr, // default is no cta-group
993 args: adaptor.getPredicate());
994 return success();
995 }
996};
997
998struct NVGPUTmaAsyncStoreOpLowering
999 : public MBarrierBasePattern<nvgpu::TmaAsyncStoreOp> {
1000 using MBarrierBasePattern<nvgpu::TmaAsyncStoreOp>::MBarrierBasePattern;
1001 LogicalResult
1002 matchAndRewrite(nvgpu::TmaAsyncStoreOp op, OpAdaptor adaptor,
1003 ConversionPatternRewriter &rewriter) const override {
1004 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1005 auto srcMemrefType = cast<MemRefType>(Val: op.getSrc().getType());
1006 Value dest = getStridedElementPtr(rewriter, loc: op->getLoc(), type: srcMemrefType,
1007 memRefDesc: adaptor.getSrc(), indices: {});
1008 SmallVector<Value> coords = adaptor.getCoordinates();
1009 for (auto [index, value] : llvm::enumerate(First&: coords)) {
1010 coords[index] = truncToI32(b, value);
1011 }
1012
1013 // TODO: Enhance the NVGPU Op for other modes too
1014 rewriter.replaceOpWithNewOp<NVVM::CpAsyncBulkTensorSharedCTAToGlobalOp>(
1015 op, args: adaptor.getTensorMapDescriptor(), args&: dest, args&: coords, args: Value{},
1016 args: NVVM::TMAStoreMode::TILE, // default is TILE mode
1017 args: adaptor.getPredicate());
1018 return success();
1019 }
1020};
1021
1022struct NVGPUGenerateWarpgroupDescriptorLowering
1023 : public ConvertOpToLLVMPattern<nvgpu::WarpgroupGenerateDescriptorOp> {
1024 using ConvertOpToLLVMPattern<
1025 nvgpu::WarpgroupGenerateDescriptorOp>::ConvertOpToLLVMPattern;
1026
1027 LogicalResult
1028 matchAndRewrite(nvgpu::WarpgroupGenerateDescriptorOp op, OpAdaptor adaptor,
1029 ConversionPatternRewriter &rewriter) const override {
1030
1031 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1032
1033 nvgpu::TensorMapSwizzleKind swizzleKind =
1034 op.getTensorMap().getType().getSwizzle();
1035
1036 unsigned layout =
1037 (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_128B) ? 128
1038 : (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_64B) ? 64
1039 : (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_32B) ? 32
1040 : 1;
1041 unsigned swizzle =
1042 (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_128B) ? 1
1043 : (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_64B) ? 2
1044 : (swizzleKind == nvgpu::TensorMapSwizzleKind::SWIZZLE_32B) ? 3
1045 : 0;
1046
1047 auto ti64 = b.getIntegerType(width: 64);
1048 auto makeConst = [&](uint64_t index) -> Value {
1049 return LLVM::ConstantOp::create(builder&: b, res: ti64, value: b.getI64IntegerAttr(value: index));
1050 };
1051 auto shiftLeft = [&](Value value, unsigned shift) -> Value {
1052 return LLVM::ShlOp::create(builder&: b, res: ti64, lhs: value, rhs: makeConst(shift));
1053 };
1054 auto shiftRight = [&](Value value, unsigned shift) -> Value {
1055 return LLVM::LShrOp::create(builder&: b, res: ti64, lhs: value, rhs: makeConst(shift));
1056 };
1057 auto insertBit = [&](Value desc, Value val, int startBit) {
1058 return LLVM::OrOp::create(builder&: b, res: ti64, lhs: desc, rhs: shiftLeft(val, startBit));
1059 };
1060
1061 int64_t sizeN = op.getTensorMap().getType().getTensor().getDimSize(idx: 0);
1062 uint64_t strideDimVal = (layout << 3) >> exclude4LSB;
1063 uint64_t leadDimVal = (sizeN * layout) >> exclude4LSB;
1064 uint64_t offsetVal = 0;
1065
1066 Value strideDim = makeConst(strideDimVal);
1067 Value leadDim = makeConst(leadDimVal);
1068
1069 Value baseAddr = getStridedElementPtr(
1070 rewriter, loc: op->getLoc(), type: cast<MemRefType>(Val: op.getTensor().getType()),
1071 memRefDesc: adaptor.getTensor(), indices: {});
1072 Value basePtr = LLVM::PtrToIntOp::create(builder&: b, res: ti64, arg: baseAddr);
1073 // Just use 14 bits for base address
1074 Value basePtr14bit = shiftRight(shiftLeft(basePtr, 46), 50);
1075
1076 int startSwizzleBit = 62, startOffsetBit = 49, startStrideBit = 32,
1077 startLeadBit = 16, startBaseAddrBit = 0;
1078 Value dsc = makeConst(0);
1079 // // [62,64) swizzle type
1080 dsc = insertBit(dsc, makeConst(swizzle), startSwizzleBit);
1081 // // [49,52) base_offset
1082 dsc = insertBit(dsc, makeConst(offsetVal), startOffsetBit);
1083 // // [32,46) stride
1084 dsc = insertBit(dsc, strideDim, startStrideBit);
1085 // // [16,30) leading dimension
1086 dsc = insertBit(dsc, leadDim, startLeadBit);
1087 // // [0,14) start_address
1088 dsc = insertBit(dsc, basePtr14bit, startBaseAddrBit);
1089
1090 LDBG() << "Generating warpgroup.descriptor: " << "leading_off:"
1091 << leadDimVal << "\t" << "stride_off :" << strideDimVal << "\t"
1092 << "base_offset:" << offsetVal << "\t" << "layout_type:" << swizzle
1093 << " (" << nvgpu::stringifyTensorMapSwizzleKind(swizzleKind)
1094 << ")\n start_addr : " << baseAddr;
1095
1096 rewriter.replaceOp(op, newValues: dsc);
1097 return success();
1098 }
1099};
1100
1101static Value makeI64Const(ImplicitLocOpBuilder &b, int32_t index) {
1102 return LLVM::ConstantOp::create(builder&: b, res: b.getIntegerType(width: 64),
1103 value: b.getI64IntegerAttr(value: index));
1104}
1105
1106/// Returns a Value that holds data type enum that is expected by CUDA driver.
1107static Value elementTypeAsLLVMConstant(ImplicitLocOpBuilder &b, Type type) {
1108 // Enum is from CUDA driver API
1109 // https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TYPES.html
1110 enum CUtensorMapDataTypeEnum {
1111 CU_TENSOR_MAP_DATA_TYPE_UINT8 = 0,
1112 CU_TENSOR_MAP_DATA_TYPE_UINT16,
1113 CU_TENSOR_MAP_DATA_TYPE_UINT32,
1114 CU_TENSOR_MAP_DATA_TYPE_INT32,
1115 CU_TENSOR_MAP_DATA_TYPE_UINT64,
1116 CU_TENSOR_MAP_DATA_TYPE_INT64,
1117 CU_TENSOR_MAP_DATA_TYPE_FLOAT16,
1118 CU_TENSOR_MAP_DATA_TYPE_FLOAT32,
1119 CU_TENSOR_MAP_DATA_TYPE_FLOAT64,
1120 CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,
1121 CU_TENSOR_MAP_DATA_TYPE_FLOAT32_FTZ,
1122 CU_TENSOR_MAP_DATA_TYPE_TFLOAT32,
1123 CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ
1124 };
1125
1126 if (type.isUnsignedInteger(width: 8))
1127 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_UINT8);
1128 if (type.isUnsignedInteger(width: 16))
1129 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_UINT16);
1130 if (type.isUnsignedInteger(width: 32))
1131 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_UINT32);
1132 if (type.isUnsignedInteger(width: 64))
1133 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_UINT64);
1134 if (type.isSignlessInteger(width: 32))
1135 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_INT32);
1136 if (type.isSignlessInteger(width: 64))
1137 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_INT64);
1138 if (type.isF16())
1139 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_FLOAT16);
1140 if (type.isF32())
1141 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_FLOAT32);
1142 if (type.isF64())
1143 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_FLOAT64);
1144 if (type.isBF16())
1145 return makeI64Const(b, index: CU_TENSOR_MAP_DATA_TYPE_BFLOAT16);
1146
1147 llvm_unreachable("Not supported data type");
1148}
1149
1150struct NVGPUTmaCreateDescriptorOpLowering
1151 : public ConvertOpToLLVMPattern<nvgpu::TmaCreateDescriptorOp> {
1152 using ConvertOpToLLVMPattern<
1153 nvgpu::TmaCreateDescriptorOp>::ConvertOpToLLVMPattern;
1154 LogicalResult
1155 matchAndRewrite(nvgpu::TmaCreateDescriptorOp op, OpAdaptor adaptor,
1156 ConversionPatternRewriter &rewriter) const override {
1157 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1158 auto llvmPointerType = LLVM::LLVMPointerType::get(context: op->getContext());
1159 Type llvmInt64Type = IntegerType::get(context: op->getContext(), width: 64);
1160
1161 Value tensorElementType =
1162 elementTypeAsLLVMConstant(b, type: op.getTensor().getType().getElementType());
1163 auto promotedOperands = getTypeConverter()->promoteOperands(
1164 loc: b.getLoc(), opOperands: op->getOperands(), adaptorOperands: adaptor.getOperands(), builder&: b);
1165
1166 Value boxArrayPtr = LLVM::AllocaOp::create(
1167 builder&: b, resultType: llvmPointerType, elementType: llvmInt64Type, arraySize: makeI64Const(b, index: 5));
1168 for (auto [index, value] : llvm::enumerate(First: adaptor.getBoxDimensions())) {
1169 Value gep = LLVM::GEPOp::create(builder&: b, resultType: llvmPointerType, elementType: llvmPointerType,
1170 basePtr: boxArrayPtr, indices: makeI64Const(b, index));
1171 LLVM::StoreOp::create(builder&: b, value, addr: gep);
1172 }
1173
1174 nvgpu::TensorMapDescriptorType desc = op.getTensorMap().getType();
1175 // Set Arguments for the function call
1176 SmallVector<Value> arguments;
1177 arguments.push_back(Elt: promotedOperands[0]); // rank
1178 arguments.push_back(Elt: promotedOperands[1]); // descriptor
1179 arguments.push_back(Elt: tensorElementType); // data type
1180 arguments.push_back(
1181 Elt: makeI64Const(b, index: (int)desc.getInterleave())); // interleave
1182 arguments.push_back(Elt: makeI64Const(b, index: (int)desc.getSwizzle())); // swizzle
1183 arguments.push_back(Elt: makeI64Const(b, index: (int)desc.getL2promo())); // l2promo
1184 arguments.push_back(Elt: makeI64Const(b, index: (int)desc.getOob())); // oob
1185 arguments.push_back(Elt: boxArrayPtr); // box dimensions
1186
1187 // Set data types of the arguments
1188 SmallVector<Type> argTypes = {
1189 llvmInt64Type, /* int64_t tensorRank */
1190 llvmPointerType, /* ptr */
1191 llvmInt64Type, /* int64_t */
1192 llvmInt64Type, /* int64_t */
1193 llvmInt64Type, /* int64_t */
1194 llvmInt64Type, /* int64_t */
1195 llvmInt64Type, /* int64_t */
1196 llvmPointerType /* ptr */
1197 };
1198 FunctionCallBuilder hostRegisterCallBuilder = {
1199 "mgpuTensorMapEncodeTiledMemref", llvmPointerType, argTypes};
1200 Value tensorMap =
1201 hostRegisterCallBuilder.create(loc: b.getLoc(), builder&: b, arguments).getResult();
1202
1203 rewriter.replaceOp(op, newValues: tensorMap);
1204 return success();
1205 }
1206};
1207
1208struct NVGPUWarpgroupMmaOpLowering
1209 : public ConvertOpToLLVMPattern<nvgpu::WarpgroupMmaOp> {
1210 using ConvertOpToLLVMPattern<nvgpu::WarpgroupMmaOp>::ConvertOpToLLVMPattern;
1211
1212 /// This is a helper class to generate required NVVM Ops for warp-group level
1213 /// matrix multiplication.
1214 /// When the given GEMM shape is larger than the shape of
1215 /// a wgmma instruction in PTX, it can generate multiple NVVM::WgmmaMmaAsyncOp
1216 /// Op(s), group and execute them asynchronously. The class also handles
1217 /// waiting for completion and iterates through WarpgroupMatrixDescriptor to
1218 /// create descriptors for each instruction.
1219 ///
1220 /// For example this is the case when the shape of GEMM is 128x128x128
1221 ///
1222 /// nvvm.wgmma.fence.aligned
1223 ///
1224 /// nvvm.wgmma.mma.async descA, descB
1225 /// iterate(descA, descB)
1226 /// nvvm.wgmma.mma.async descA, descB
1227 /// [6x times more]
1228 ///
1229 /// nvvm.wgmma.group.sync.aligned
1230 /// nvvm.wgmma.wait.group.sync [groupId]
1231 ///
1232 class WarpgroupGemm {
1233 nvgpu::WarpgroupMmaOp op;
1234 ImplicitLocOpBuilder b;
1235 OpAdaptor adaptor;
1236
1237 // Entire shape of the given Op
1238 int64_t totalM, totalN, totalK;
1239
1240 // Shape of one wgmma instruction
1241 int wgmmaM = 0, wgmmaN = 0, wgmmaK = 0;
1242
1243 // Iteration counts for GEMM
1244 int iterationM = 0, iterationN = 0, iterationK = 0;
1245
1246 /// The function returns the shape of wgmma instruction that is defined in
1247 /// PTX programming guide.
1248 /// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-shape
1249 void findWgmmaShape(int64_t sizeM, int64_t sizeN, Type inputElemType) {
1250 wgmmaM = 64;
1251 wgmmaN = sizeN;
1252 if (inputElemType.isTF32()) {
1253 wgmmaK = 8;
1254 } else if (inputElemType.isF16() || inputElemType.isBF16()) {
1255 wgmmaK = 16;
1256 } else if (isa<Float8E4M3FNType, Float8E5M2Type>(Val: inputElemType) ||
1257 inputElemType.isInteger(width: 8)) {
1258 wgmmaK = 32;
1259 } else if (inputElemType.isInteger(width: 1)) {
1260 wgmmaK = 256;
1261 } else {
1262 llvm_unreachable("msg: not supported K shape");
1263 }
1264 LDBG() << "Generating WgmmaMmaAsyncOp shape[m = " << wgmmaM
1265 << ", n = " << wgmmaN << ", k = " << wgmmaK << "]";
1266 }
1267
1268 /// Generates WGMMATypesAttr from MLIR Type
1269 NVVM::WGMMATypesAttr generateWgmmaType(Type type,
1270 bool useF32 = false) const {
1271 auto getWgmmaType = [=](Type elemType) {
1272 if (elemType.isF32() || elemType.isTF32())
1273 return useF32 ? NVVM::WGMMATypes::f32 : NVVM::WGMMATypes::tf32;
1274 if (elemType.isF16())
1275 return NVVM::WGMMATypes::f16;
1276 if (elemType.isBF16())
1277 return NVVM::WGMMATypes::bf16;
1278 if (isa<Float8E4M3FNType>(Val: elemType))
1279 return NVVM::WGMMATypes::e4m3;
1280 if (isa<Float8E5M2Type>(Val: elemType))
1281 return NVVM::WGMMATypes::e5m2;
1282 if (elemType.isInteger(width: 1))
1283 return NVVM::WGMMATypes::b1;
1284 if (elemType.isInteger(width: 8))
1285 return NVVM::WGMMATypes::s8;
1286 if (elemType.isUnsignedInteger(width: 8))
1287 return NVVM::WGMMATypes::u8;
1288 if (elemType.isInteger(width: 32))
1289 return NVVM::WGMMATypes::s32;
1290 llvm_unreachable("unsupported type");
1291 };
1292 return NVVM::WGMMATypesAttr::get(context: op->getContext(), value: getWgmmaType(type));
1293 }
1294
1295 /// Generates layout attribute for the input matrix for wgmma instruction
1296 NVVM::MMALayoutAttr
1297 generateWgmmaLayout(std::optional<bool> transpose) const {
1298 if (transpose.value_or(u: false))
1299 return NVVM::MMALayoutAttr::get(context: op->getContext(), value: NVVM::MMALayout::col);
1300 return NVVM::MMALayoutAttr::get(context: op->getContext(), value: NVVM::MMALayout::row);
1301 }
1302
1303 /// Generates shape attribute for wgmma instruction
1304 NVVM::MMAShapeAttr generateWgmmaShape() const {
1305 return NVVM::MMAShapeAttr::get(context: op->getContext(), m: wgmmaM, n: wgmmaN, k: wgmmaK);
1306 }
1307
1308 /// Generates scale attributes of output matrix for wgmma instruction
1309 NVVM::WGMMAScaleOutAttr generateScaleOut() const {
1310 return NVVM::WGMMAScaleOutAttr::get(context: op->getContext(),
1311 value: NVVM::WGMMAScaleOut::one);
1312 }
1313 /// Generates scale attributes of input matrix for wgmma instruction
1314 NVVM::WGMMAScaleInAttr generateScaleIn() const {
1315 return NVVM::WGMMAScaleInAttr::get(context: op->getContext(),
1316 value: NVVM::WGMMAScaleIn::one);
1317 }
1318
1319 /// Basic function to generate Add
1320 Value makeAdd(Value lhs, Value rhs) {
1321 return LLVM::AddOp::create(builder&: b, res: lhs.getType(), lhs, rhs);
1322 };
1323
1324 /// Moves the descriptor pointer of matrix-A for the next wgmma instruction.
1325 /// Currently, it only handles row-major.
1326 ///
1327 /// It moves the pointer like below for [128][64] size:
1328 /// +2 +4 +6
1329 /// ↓ ↓ ↓
1330 /// descA ---> +--+--+--+--+
1331 /// |->|->|->|->|
1332 /// | | | | |
1333 /// | | | | |
1334 /// | | | | |
1335 /// descA+512---> +-----------+
1336 /// | | | | |
1337 /// | | | | |
1338 /// | | | | |
1339 /// | | | | |
1340 /// +-----------+
1341 ///
1342 Value iterateDescriptorA(Value desc, int i, int j, int k) {
1343 MemRefType matrixTypeA = op.getDescriptorA().getType().getTensor();
1344 Type elemA = matrixTypeA.getElementType();
1345 int byte = elemA.getIntOrFloatBitWidth() / 8;
1346 int tileShapeA = matrixTypeA.getDimSize(idx: 1);
1347 int incrementVal = ((wgmmaK * k) + (totalK * tileShapeA * i)) * byte;
1348 incrementVal = incrementVal >> exclude4LSB;
1349 LDBG() << "\t\t[m: " << i << " n: " << j << " k: " << k
1350 << "] [wgmma descriptors] Descriptor A + " << incrementVal
1351 << " | \t ";
1352 if (!incrementVal)
1353 return desc;
1354 return makeAdd(lhs: desc, rhs: makeI64Const(b, index: incrementVal));
1355 }
1356
1357 /// Moves the descriptor pointer of matrix-B for the next wgmma instruction.
1358 /// Currently, it only handles column-major.
1359 ///
1360 /// It moves the pointer like below for [128][64] size:
1361 /// descB ---> +--+--+--+--+--+--+--+--+
1362 /// |↓ | | | | | | | |
1363 /// |↓ | | | | | | | |
1364 /// |↓ | | | | | | | |
1365 /// |↓ | | | | | | | |
1366 /// +--+--+--+--+--+--+--+--+
1367 ///
1368 Value iterateDescriptorB(Value desc, int i, int j, int k) {
1369 MemRefType matrixTypeB = op.getDescriptorB().getType().getTensor();
1370 Type elemB = matrixTypeB.getElementType();
1371 int byte = elemB.getIntOrFloatBitWidth() / 8;
1372 int incrementVal = matrixTypeB.getDimSize(idx: 0) * wgmmaK * k * byte;
1373 incrementVal = incrementVal >> exclude4LSB;
1374 LDBG() << "Descriptor B + " << incrementVal;
1375 if (!incrementVal)
1376 return desc;
1377 return makeAdd(lhs: desc, rhs: makeI64Const(b, index: incrementVal));
1378 }
1379
1380 /// This function generates a WgmmaMmaAsyncOp using provided GMMA matrix
1381 /// descriptors and arranges them based on induction variables: i, j, and k.
1382 Value generateWgmma(int i, int j, int k, Value matrixC) {
1383 LDBG() << "\t wgmma." << "m" << wgmmaM << "n" << wgmmaN << "k" << wgmmaK
1384 << "(A[" << (iterationM * wgmmaM) << ":"
1385 << (iterationM * wgmmaM) + wgmmaM << "][" << (iterationK * wgmmaK)
1386 << ":" << (iterationK * wgmmaK + wgmmaK) << "] * " << " B["
1387 << (iterationK * wgmmaK) << ":" << (iterationK * wgmmaK + wgmmaK)
1388 << "][" << 0 << ":" << wgmmaN << "])";
1389
1390 Value descriptorA = iterateDescriptorA(desc: adaptor.getDescriptorA(), i, j, k);
1391 Value descriptorB = iterateDescriptorB(desc: adaptor.getDescriptorB(), i, j, k);
1392
1393 Type elemA = op.getDescriptorA().getType().getTensor().getElementType();
1394 NVVM::WGMMATypesAttr itypeA = generateWgmmaType(type: elemA);
1395
1396 Type elemB = op.getDescriptorB().getType().getTensor().getElementType();
1397 NVVM::WGMMATypesAttr itypeB = generateWgmmaType(type: elemB);
1398
1399 Type elemD = op.getMatrixC().getType().getFragmented().getElementType();
1400 NVVM::WGMMATypesAttr itypeD = generateWgmmaType(type: elemD, useF32: true);
1401
1402 NVVM::MMAShapeAttr shape = generateWgmmaShape();
1403 NVVM::WGMMAScaleOutAttr scaleOut = generateScaleOut();
1404 NVVM::WGMMAScaleInAttr scaleIn = generateScaleIn();
1405 NVVM::MMALayoutAttr layoutA = generateWgmmaLayout(transpose: op.getTransposeA());
1406 NVVM::MMALayoutAttr layoutB = generateWgmmaLayout(transpose: !op.getTransposeB());
1407
1408 auto overflow = NVVM::MMAIntOverflowAttr::get(
1409 context: op->getContext(), value: NVVM::MMAIntOverflow::wrapped);
1410
1411 return NVVM::WgmmaMmaAsyncOp::create(
1412 builder&: b, results: matrixC.getType(), inouts: matrixC, descriptorA, descriptorB, shape,
1413 typeA: itypeA, typeB: itypeB, typeD: itypeD, scaleD: scaleOut, scaleA: scaleIn, scaleB: scaleIn, layoutA, layoutB,
1414 satfinite: overflow);
1415 }
1416
1417 /// Generates multiple wgmma instructions to complete the given GEMM shape
1418 Value generateWgmmaGroup() {
1419 Value wgmmaResult =
1420 LLVM::PoisonOp::create(builder&: b, res: adaptor.getMatrixC().getType());
1421
1422 // Perform GEMM
1423 SmallVector<Value> wgmmaResults;
1424 for (int i = 0; i < iterationM; ++i) {
1425 Value matrixC =
1426 LLVM::ExtractValueOp::create(builder&: b, container: adaptor.getMatrixC(), position: i);
1427 for (int j = 0; j < iterationN; ++j)
1428 for (int k = 0; k < iterationK; ++k)
1429 matrixC = generateWgmma(i, j, k, matrixC);
1430 wgmmaResults.push_back(Elt: matrixC);
1431 }
1432 for (auto [idx, matrix] : llvm::enumerate(First&: wgmmaResults)) {
1433 wgmmaResult = LLVM::InsertValueOp::create(builder&: b, res: wgmmaResult.getType(),
1434 container: wgmmaResult, value: matrix, position: idx);
1435 }
1436 return wgmmaResult;
1437 }
1438
1439 public:
1440 WarpgroupGemm(nvgpu::WarpgroupMmaOp op, ImplicitLocOpBuilder &b,
1441 OpAdaptor adaptor)
1442 : op(op), b(b), adaptor(adaptor) {
1443 // Find the entire GEMM Shape
1444 totalM = op.getDescriptorA().getType().getTensor().getDimSize(idx: 0);
1445 totalN = op.getDescriptorB().getType().getTensor().getDimSize(idx: 1);
1446 totalK = op.getDescriptorA().getType().getTensor().getDimSize(idx: 1);
1447 LDBG() << "===--- GEMM D[" << totalM << "][" << totalN << "] += A["
1448 << totalM << "][" << totalK << "] * B[" << totalK << "][" << totalN
1449 << "] ---===";
1450
1451 // Find the shape for one wgmma instruction
1452 findWgmmaShape(
1453 sizeM: totalM, sizeN: totalN,
1454 inputElemType: op.getDescriptorA().getType().getTensor().getElementType());
1455
1456 // Iterations counts to complete the given shape with wgmma shape
1457 iterationM = totalM / wgmmaM;
1458 iterationN = totalN / wgmmaN;
1459 iterationK = totalK / wgmmaK;
1460 }
1461
1462 /// Generates WgmmaMmaAsync Ops to complete the specified GEMM shape. It
1463 /// includes generating a fence Op (WgmmaFenceAlignedOp) before the
1464 /// instructions and group synchronization, as well as waiting
1465 /// (WgmmaGroupSyncAlignedOp) for group synchronization
1466 /// (WgmmaWaitGroupSyncOp) after the instructions.
1467 Value generateWarpgroupMma() {
1468 NVVM::WgmmaFenceAlignedOp::create(builder&: b);
1469 Value wgmmaResult = generateWgmmaGroup();
1470 NVVM::WgmmaGroupSyncAlignedOp::create(builder&: b);
1471 NVVM::WgmmaWaitGroupSyncOp::create(builder&: b, group: op.getWaitGroup());
1472 return wgmmaResult;
1473 }
1474 };
1475 LogicalResult
1476 matchAndRewrite(nvgpu::WarpgroupMmaOp op, OpAdaptor adaptor,
1477 ConversionPatternRewriter &rewriter) const override {
1478 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1479
1480 // Step 1. Build a helper class
1481 WarpgroupGemm warpgroupGemm(op, b, adaptor);
1482
1483 // Step 2. Get the entire GEMM Shape
1484 Value wgmmaResult = warpgroupGemm.generateWarpgroupMma();
1485
1486 // Step 3. Replace fragmented result struct with the op results
1487 rewriter.replaceOp(op, newValues: wgmmaResult);
1488 return success();
1489 }
1490};
1491
1492struct NVGPUWarpgroupMmaStoreOpLowering
1493 : public ConvertOpToLLVMPattern<nvgpu::WarpgroupMmaStoreOp> {
1494 using ConvertOpToLLVMPattern<
1495 nvgpu::WarpgroupMmaStoreOp>::ConvertOpToLLVMPattern;
1496
1497 /// This function stores a fragmented register matrix owned by a warp group
1498 /// (128 threads) into a memref. Each thread has 64 registers, each the size
1499 /// of a struct.
1500 /// Here is what each threads (T) holds, each `d` is struct value with a
1501 /// number.
1502 ///
1503 /// Threads in warp-group (128 threads) and what they owns in the matrixD:
1504 /// 0-31 Warp-0 -> MatrixD[0:15 ][0:N]
1505 /// 32-63 Warp-1 -> MatrixD[16:31][0:N]
1506 /// 64-95 Warp-2 -> MatrixD[32:47][0:N]
1507 /// 96-127 Warp-3 -> MatrixD[48:64][0:N]
1508 ///
1509 /// Matrix-D:
1510 /// +______________________________________________________________________+
1511 /// | 0-1 | 2-3 | 4-5 | 6-7 | 8-9 | 10-11|..|N-8,N-7 |
1512 /// 0 | T0:d0-d1 |T1:d0-d1 |T2:d0-d1 |T3:d0-d1 |T0:d4-d5| T1:d4-d5..|T0:dX-dY|
1513 /// 1 | T4:d0-d1 |T5:d0-d1 |T6:d0-d1 |T7:d0-d1 |T4:d4-d5| T5:d4-d5..|T4:dX-dY|
1514 /// ..| .........|.........|.........|.........|........|...........|........|
1515 /// 8 | T0:d2-d3 |T1:d2-d3 |T2:d2-d3 |T3:d2-d3 |T0:d6-d7|T1:d6-d7,..|T0:dZ-dW|
1516 /// 9 | T4:d2-d3 |T5:d2-d3 |T6:d2-d3 |T7:d2-d3 |T4:d6-d7| T5:d6-d7..|T4:dZ-dW|
1517 /// ..| .........|.........|.........|.........|........|...........|........|
1518 /// 15| T28:d2-d3|T29:d2-d3|T30:d2-d3|T31:d2-d3|........|...........|........|
1519 /// 16| T32:d2-d3|T33:d2-d3|T34:d2-d3|T35:d2-d3|........|...........|........|
1520 /// ..| .........|.........|.........|.........|........|...........|........|
1521 /// 32| T64:d2-d3|T65:d2-d3|T66:d2-d3|T67:d2-d3|........|...........|........|
1522 /// ..| .........|.........|.........|.........|........|...........|........|
1523 /// 48| T96:d2-d3|T97:d2-d3|T98:d2-d3|T99:d2-d3|........|...........|........|
1524 /// ..| .........|.........|.........|.........|........|...........|........|
1525 /// +______________________________________________________________________+
1526 ///
1527 /// \param rewriter: The pattern rewriter.
1528 /// \param matrixD: Result of the warp-group MMA operation (fragmented
1529 /// matrix). It is holded by a thread and a struct with 64 elements.
1530 /// \param dstMemref: The memref where the registers will be stored.
1531 /// \param offset: the offset within the memref where the registers will be
1532 /// stored.
1533 void storeFragmentedMatrix(ImplicitLocOpBuilder &b, Value matrixD,
1534 TypedValue<MemRefType> dstMemref,
1535 int offset) const {
1536 Type i32 = b.getI32Type();
1537
1538 auto makeConst = [&](int32_t index) -> Value {
1539 return LLVM::ConstantOp::create(builder&: b, res: i32, value: b.getI32IntegerAttr(value: index));
1540 };
1541 Value c1 = makeConst(1);
1542 Value c2 = makeConst(2);
1543 Value c4 = makeConst(4);
1544 Value c8 = makeConst(8);
1545 Value c16 = makeConst(16);
1546 Value warpSize = makeConst(kWarpSize);
1547
1548 auto makeMul = [&](Value lhs, Value rhs) -> Value {
1549 return LLVM::MulOp::create(builder&: b, res: lhs.getType(), lhs, rhs);
1550 };
1551 auto makeAdd = [&](Value lhs, Value rhs) -> Value {
1552 return LLVM::AddOp::create(builder&: b, res: lhs.getType(), lhs, rhs);
1553 };
1554
1555 auto makeExtractAndStore = [&](int i, Value wgmmaResult, Value x, Value y,
1556 TypedValue<::mlir::MemRefType> memref) {
1557 Type it = b.getIndexType();
1558 Value idx = arith::IndexCastOp::create(builder&: b, out: it, in: x);
1559 Value idy0 = arith::IndexCastOp::create(builder&: b, out: it, in: y);
1560 Value idy1 = arith::IndexCastOp::create(builder&: b, out: it, in: makeAdd(y, c1));
1561 Value d0 = LLVM::ExtractValueOp::create(builder&: b, container: wgmmaResult, position: i);
1562 Value d1 = LLVM::ExtractValueOp::create(builder&: b, container: wgmmaResult, position: i + 1);
1563 memref::StoreOp::create(builder&: b, valueToStore: d0, memref, indices: ValueRange{idx, idy0});
1564 memref::StoreOp::create(builder&: b, valueToStore: d1, memref, indices: ValueRange{idx, idy1});
1565 };
1566
1567 Value tidx = NVVM::ThreadIdXOp::create(builder&: b, resultType: i32);
1568 Value laneId = LLVM::URemOp::create(builder&: b, res: i32, lhs: tidx, rhs: warpSize);
1569 Value warpId = LLVM::UDivOp::create(builder&: b, res: i32, lhs: tidx, rhs: warpSize);
1570 Value lane4Id = LLVM::UDivOp::create(builder&: b, res: i32, lhs: laneId, rhs: c4);
1571 Value lane4modId = LLVM::URemOp::create(builder&: b, res: i32, lhs: laneId, rhs: c4);
1572
1573 Value tj = makeMul(lane4modId, c2);
1574 Value ti = makeAdd(lane4Id, makeMul(warpId, c16));
1575 if (offset)
1576 ti = makeAdd(ti, makeConst(offset));
1577
1578 auto structType = cast<LLVM::LLVMStructType>(Val: matrixD.getType());
1579
1580 // Number of 32-bit registers owns per thread
1581 constexpr unsigned numAdjacentRegisters = 2;
1582 // Number of 8x8 matrices one below another per warp
1583 constexpr unsigned numStackedMatrices = 2;
1584
1585 size_t storeCount = (structType.getBody().size() /
1586 (numStackedMatrices * numAdjacentRegisters));
1587
1588 for (size_t i = 0; i < numStackedMatrices; ++i) {
1589 Value idx = makeAdd(ti, makeMul(makeConst(i), c8));
1590 for (size_t j = 0; j < storeCount; ++j) {
1591 Value idy = makeAdd(tj, makeMul(makeConst(j), c8));
1592 size_t structIndex = (i * numAdjacentRegisters) +
1593 (j * (numStackedMatrices * numAdjacentRegisters));
1594 makeExtractAndStore(structIndex, matrixD, idx, idy, dstMemref);
1595 }
1596 }
1597 }
1598
1599 LogicalResult
1600 matchAndRewrite(nvgpu::WarpgroupMmaStoreOp op, OpAdaptor adaptor,
1601 ConversionPatternRewriter &rewriter) const override {
1602 int offset = 0;
1603 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1604 Value matriDValue = adaptor.getMatrixD();
1605 auto stype = cast<LLVM::LLVMStructType>(Val: matriDValue.getType());
1606 for (auto [idx, matrixD] : llvm::enumerate(First: stype.getBody())) {
1607 auto structType = cast<LLVM::LLVMStructType>(Val: matrixD);
1608 Value innerStructValue =
1609 LLVM::ExtractValueOp::create(builder&: b, container: matriDValue, position: idx);
1610 storeFragmentedMatrix(b, matrixD: innerStructValue, dstMemref: op.getDstMemref(), offset);
1611 offset += structType.getBody().size();
1612 }
1613 rewriter.eraseOp(op);
1614 return success();
1615 }
1616};
1617
1618struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
1619 : public ConvertOpToLLVMPattern<nvgpu::WarpgroupMmaInitAccumulatorOp> {
1620 using ConvertOpToLLVMPattern<
1621 nvgpu::WarpgroupMmaInitAccumulatorOp>::ConvertOpToLLVMPattern;
1622 LogicalResult
1623 matchAndRewrite(nvgpu::WarpgroupMmaInitAccumulatorOp op, OpAdaptor adaptor,
1624 ConversionPatternRewriter &rewriter) const override {
1625 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1626 LLVM::LLVMStructType packStructType = cast<LLVM::LLVMStructType>(
1627 Val: getTypeConverter()->convertType(t: op.getMatrixC().getType()));
1628 Type elemType = cast<LLVM::LLVMStructType>(Val: packStructType.getBody().front())
1629 .getBody()
1630 .front();
1631 Value zero = LLVM::ConstantOp::create(builder&: b, res: elemType, value: b.getZeroAttr(type: elemType));
1632 Value packStruct = LLVM::PoisonOp::create(builder&: b, res: packStructType);
1633 SmallVector<Value> innerStructs;
1634 // Unpack the structs and set all values to zero
1635 for (auto [idx, s] : llvm::enumerate(First: packStructType.getBody())) {
1636 auto structType = cast<LLVM::LLVMStructType>(Val: s);
1637 Value structValue = LLVM::ExtractValueOp::create(builder&: b, container: packStruct, position: idx);
1638 for (unsigned i = 0; i < structType.getBody().size(); ++i) {
1639 structValue = LLVM::InsertValueOp::create(builder&: b, res: structType, container: structValue,
1640 value: zero, position: ArrayRef<int64_t>({i}));
1641 }
1642 innerStructs.push_back(Elt: structValue);
1643 }
1644 // Pack the inner structs into a single struct
1645 for (auto [idx, matrix] : llvm::enumerate(First&: innerStructs)) {
1646 packStruct = LLVM::InsertValueOp::create(builder&: b, res: packStruct.getType(),
1647 container: packStruct, value: matrix, position: idx);
1648 }
1649 rewriter.replaceOp(op, newValues: packStruct);
1650 return success();
1651 }
1652};
1653
1654struct NVGPUTmaFenceOpLowering
1655 : public ConvertOpToLLVMPattern<nvgpu::TmaFenceOp> {
1656 using ConvertOpToLLVMPattern<nvgpu::TmaFenceOp>::ConvertOpToLLVMPattern;
1657 LogicalResult
1658 matchAndRewrite(nvgpu::TmaFenceOp op, OpAdaptor adaptor,
1659 ConversionPatternRewriter &rewriter) const override {
1660 MLIRContext *ctx = op.getContext();
1661 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1662 auto i32Ty = b.getI32Type();
1663 Value tensormapSize =
1664 LLVM::ConstantOp::create(builder&: b, res: i32Ty, value: rewriter.getI32IntegerAttr(value: 128));
1665
1666 auto memscope =
1667 NVVM::MemScopeKindAttr::get(context: ctx, value: ::mlir::NVVM::MemScopeKind::SYS);
1668
1669 rewriter.replaceOpWithNewOp<NVVM::FenceProxyAcquireOp>(
1670 op, args&: memscope, args: adaptor.getTensorMapDescriptor(), args&: tensormapSize);
1671
1672 return success();
1673 }
1674};
1675
1676struct NVGPUTmaPrefetchOpLowering
1677 : public ConvertOpToLLVMPattern<nvgpu::TmaPrefetchOp> {
1678 using ConvertOpToLLVMPattern<nvgpu::TmaPrefetchOp>::ConvertOpToLLVMPattern;
1679 LogicalResult
1680 matchAndRewrite(nvgpu::TmaPrefetchOp op, OpAdaptor adaptor,
1681 ConversionPatternRewriter &rewriter) const override {
1682 rewriter.replaceOpWithNewOp<NVVM::PrefetchOp>(
1683 op, /* CacheLevel */ args: nullptr, /* Cache Eviction Priority */ args: nullptr,
1684 args: adaptor.getTensorMapDescriptor(), args: adaptor.getPredicate(),
1685 /* Tensormap UnitAttr */ args: mlir::UnitAttr::get(context: op.getContext()));
1686 return success();
1687 }
1688};
1689
1690struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
1691 using ConvertOpToLLVMPattern<nvgpu::RcpOp>::ConvertOpToLLVMPattern;
1692 LogicalResult
1693 matchAndRewrite(nvgpu::RcpOp op, OpAdaptor adaptor,
1694 ConversionPatternRewriter &rewriter) const override {
1695 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1696 auto i64Ty = b.getI64Type();
1697 auto f32Ty = b.getF32Type();
1698 VectorType inTy = op.getIn().getType();
1699 // apply rcp.approx.ftz.f on each element in vector.
1700 auto convert1DVec = [&](Type llvm1DVectorTy, Value inVec) {
1701 Value ret1DVec = LLVM::PoisonOp::create(builder&: b, res: llvm1DVectorTy);
1702 int numElems = llvm::cast<VectorType>(Val&: llvm1DVectorTy).getNumElements();
1703 for (int i = 0; i < numElems; i++) {
1704 Value idx = LLVM::ConstantOp::create(builder&: b, res: i64Ty, value: b.getI64IntegerAttr(value: i));
1705 Value elem = LLVM::ExtractElementOp::create(builder&: b, vector: inVec, position: idx);
1706 Value dst = NVVM::RcpApproxFtzF32Op::create(builder&: b, res: f32Ty, arg: elem);
1707 ret1DVec = LLVM::InsertElementOp::create(builder&: b, vector: ret1DVec, value: dst, position: idx);
1708 }
1709 return ret1DVec;
1710 };
1711 if (inTy.getRank() == 1) {
1712 rewriter.replaceOp(op, newValues: convert1DVec(inTy, adaptor.getIn()));
1713 return success();
1714 }
1715 return LLVM::detail::handleMultidimensionalVectors(
1716 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *(this->getTypeConverter()),
1717 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
1718 OpAdaptor adaptor(operands);
1719 return convert1DVec(llvm1DVectorTy, adaptor.getIn());
1720 },
1721 rewriter);
1722 }
1723};
1724
1725//===----------------------------------------------------------------------===//
1726// NVGPUTruncfOp Lowering
1727//===----------------------------------------------------------------------===//
1728
1729enum class FPKind { F32, BF16, F16, F8, F6, F4 };
1730
1731/// Get the effective bit width of a floating-point type.
1732/// f6 types are 6-bit but NVVM Ops expect 8-bit (i8) containers.
1733static int getEffectiveBitWidth(int bitWidth) {
1734 return bitWidth == 6 ? 8 : bitWidth;
1735}
1736
1737static std::optional<FPKind> classifyFPType(Type t) {
1738 static constexpr auto isConvertibleF8Type = [](Type t) {
1739 return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(Val: t);
1740 };
1741 static constexpr auto isConvertibleF6Type = [](Type t) {
1742 return isa<Float6E2M3FNType, Float6E3M2FNType>(Val: t);
1743 };
1744 static constexpr auto isConvertibleF4Type = [](Type t) {
1745 return isa<Float4E2M1FNType>(Val: t);
1746 };
1747
1748 if (t.isF32())
1749 return FPKind::F32;
1750 if (t.isBF16())
1751 return FPKind::BF16;
1752 if (t.isF16())
1753 return FPKind::F16;
1754 if (isConvertibleF8Type(t))
1755 return FPKind::F8;
1756 if (isConvertibleF6Type(t))
1757 return FPKind::F6;
1758 if (isConvertibleF4Type(t))
1759 return FPKind::F4;
1760
1761 return std::nullopt;
1762}
1763
1764/// Conversion op identifier for nvgpu.truncf lowering dispatch table.
1765enum class FPTruncConvOp {
1766 F32x2_TO_F16x2,
1767 F32x2_TO_BF16x2,
1768 F32x2_TO_F8x2,
1769 F32x2_TO_F6x2,
1770 F32x2_TO_F4x2,
1771 F16x2_TO_F8x2,
1772 F16x2_TO_F6x2,
1773 F16x2_TO_F4x2,
1774 BF16x2_TO_F8x2,
1775 BF16x2_TO_F6x2,
1776 BF16x2_TO_F4x2,
1777};
1778
1779struct FPTruncTableEntry {
1780 FPKind src;
1781 FPKind dst;
1782 FPTruncConvOp convOp;
1783};
1784
1785static constexpr FPTruncTableEntry kFPTruncTable[] = {
1786 // f32 source
1787 {.src: FPKind::F32, .dst: FPKind::F16, .convOp: FPTruncConvOp::F32x2_TO_F16x2},
1788 {.src: FPKind::F32, .dst: FPKind::BF16, .convOp: FPTruncConvOp::F32x2_TO_BF16x2},
1789 {.src: FPKind::F32, .dst: FPKind::F8, .convOp: FPTruncConvOp::F32x2_TO_F8x2},
1790 {.src: FPKind::F32, .dst: FPKind::F6, .convOp: FPTruncConvOp::F32x2_TO_F6x2},
1791 {.src: FPKind::F32, .dst: FPKind::F4, .convOp: FPTruncConvOp::F32x2_TO_F4x2},
1792 // f16 source
1793 {.src: FPKind::F16, .dst: FPKind::F8, .convOp: FPTruncConvOp::F16x2_TO_F8x2},
1794 {.src: FPKind::F16, .dst: FPKind::F6, .convOp: FPTruncConvOp::F16x2_TO_F6x2},
1795 {.src: FPKind::F16, .dst: FPKind::F4, .convOp: FPTruncConvOp::F16x2_TO_F4x2},
1796 // bf16 source
1797 {.src: FPKind::BF16, .dst: FPKind::F8, .convOp: FPTruncConvOp::BF16x2_TO_F8x2},
1798 {.src: FPKind::BF16, .dst: FPKind::F6, .convOp: FPTruncConvOp::BF16x2_TO_F6x2},
1799 {.src: FPKind::BF16, .dst: FPKind::F4, .convOp: FPTruncConvOp::BF16x2_TO_F4x2},
1800};
1801
1802/// Find the conversion table entry whose source/destination `FPKind`s match the
1803/// given element types.
1804template <typename TableEntry, size_t N>
1805static std::optional<TableEntry>
1806lookupConvOp(const TableEntry (&table)[N], Type srcElemType, Type dstElemType) {
1807 std::optional<FPKind> srcKind = classifyFPType(t: srcElemType);
1808 std::optional<FPKind> dstKind = classifyFPType(t: dstElemType);
1809 if (!srcKind || !dstKind)
1810 return std::nullopt;
1811 for (const TableEntry &entry : table) {
1812 if (entry.src == *srcKind && entry.dst == *dstKind)
1813 return entry;
1814 }
1815 return std::nullopt;
1816}
1817
1818/// Extract a single element from a vector.
1819static Value extractElement(ImplicitLocOpBuilder &b, Value srcVec, int idx) {
1820 assert(idx >= 0 &&
1821 idx < cast<VectorType>(srcVec.getType()).getNumElements() &&
1822 "extractElement: index out of bounds");
1823 IntegerType i64Ty = b.getI64Type();
1824 return b.create<LLVM::ExtractElementOp>(
1825 args&: srcVec, args: b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: idx)));
1826}
1827
1828/// Extract a pair of f32 values from an i32 vector at the given base index.
1829static std::pair<Value, Value> extractF32Pair(ImplicitLocOpBuilder &b,
1830 Value srcI32Vec, int baseIdx) {
1831 FloatType f32Ty = b.getF32Type();
1832 Value elem0 = extractElement(b, srcVec: srcI32Vec, idx: baseIdx);
1833 Value elem1 = extractElement(b, srcVec: srcI32Vec, idx: baseIdx + 1);
1834 return {b.create<LLVM::BitcastOp>(args&: f32Ty, args&: elem0),
1835 b.create<LLVM::BitcastOp>(args&: f32Ty, args&: elem1)};
1836}
1837
1838/// Extract a vector of elements of size i32 from an i32 vector and bitcast to
1839/// the specified vector type.
1840static Value extractAndBitcast(ImplicitLocOpBuilder &b, Value srcI32Vec,
1841 int idx, VectorType vecTy) {
1842 Value elem = extractElement(b, srcVec: srcI32Vec, idx);
1843 return b.create<LLVM::BitcastOp>(args&: vecTy, args&: elem);
1844}
1845
1846/// Create a sub-byte conversion from an f32 pair source and return the native
1847/// result.
1848template <typename ConvertOp, typename... Args>
1849static Value convertFromF32Pair(ImplicitLocOpBuilder &b, Value srcI32Vec,
1850 int srcBaseIdx, Type resultTy, Args &&...args) {
1851 auto [lo, hi] = extractF32Pair(b, srcI32Vec, baseIdx: srcBaseIdx);
1852 return b.create<ConvertOp>(resultTy, hi, lo, /*scale_factor=*/Value(),
1853 std::forward<Args>(args)...);
1854}
1855
1856/// Create a sub-byte conversion from a packed f16x2/bf16x2 source and return
1857/// the native result.
1858template <typename ConvertOp, typename... Args>
1859static Value convertFromPacked(ImplicitLocOpBuilder &b, Value srcI32Vec,
1860 int srcBaseIdx, Type srcElemTy, Type resultTy,
1861 Args &&...args) {
1862 Value src = extractAndBitcast(b, srcI32Vec, idx: srcBaseIdx,
1863 vecTy: VectorType::get(shape: 2, elementType: srcElemTy));
1864 return b.create<ConvertOp>(resultTy, src, /*scale_factor=*/Value(),
1865 std::forward<Args>(args)...);
1866}
1867
1868/// Create a typed NVVM truncation conversion.
1869static Value createTruncConversion(
1870 ImplicitLocOpBuilder &b, MLIRContext *ctx, FPTruncConvOp convOp,
1871 Value srcI32Vec, int srcBaseIdx, NVVM::FPRoundingModeAttr rndAttr,
1872 NVVM::SaturationModeAttr satAttr, BoolAttr reluAttr, Type dstElemType,
1873 Type actualDstFloatType, Value randomBits = Value()) {
1874 IntegerType i8Ty = b.getI8Type();
1875 IntegerType i16Ty = b.getI16Type();
1876 IntegerType i32Ty = b.getI32Type();
1877 TypeAttr dstTyAttr = TypeAttr::get(type: dstElemType);
1878 TypeAttr actualDstTyAttr = TypeAttr::get(type: actualDstFloatType);
1879
1880 switch (convOp) {
1881 case FPTruncConvOp::F32x2_TO_F16x2: {
1882 auto [lo, hi] = extractF32Pair(b, srcI32Vec, baseIdx: srcBaseIdx);
1883 Value r = b.create<NVVM::ConvertF32x2ToF16x2Op>(
1884 args: VectorType::get(shape: 2, elementType: b.getF16Type()), args&: hi, args&: lo, args&: randomBits, args&: rndAttr,
1885 args&: satAttr, args&: reluAttr);
1886 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
1887 }
1888 case FPTruncConvOp::F32x2_TO_BF16x2: {
1889 auto [lo, hi] = extractF32Pair(b, srcI32Vec, baseIdx: srcBaseIdx);
1890 Value r = b.create<NVVM::ConvertF32x2ToBF16x2Op>(
1891 args: VectorType::get(shape: 2, elementType: b.getBF16Type()), args&: hi, args&: lo, args&: randomBits, args&: rndAttr,
1892 args&: satAttr, args&: reluAttr);
1893 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
1894 }
1895 case FPTruncConvOp::F32x2_TO_F8x2:
1896 return convertFromF32Pair<NVVM::ConvertF32x2ToF8x2Op>(
1897 b, srcI32Vec, srcBaseIdx, resultTy: i16Ty, args&: rndAttr, args&: satAttr, args&: reluAttr, args&: dstTyAttr);
1898 case FPTruncConvOp::F32x2_TO_F6x2:
1899 return convertFromF32Pair<NVVM::ConvertF32x2ToF6x2Op>(
1900 b, srcI32Vec, srcBaseIdx, resultTy: i16Ty, args&: reluAttr, args&: actualDstTyAttr);
1901 case FPTruncConvOp::F32x2_TO_F4x2:
1902 return convertFromF32Pair<NVVM::ConvertF32x2ToF4x2Op>(
1903 b, srcI32Vec, srcBaseIdx, resultTy: i8Ty, args&: reluAttr, args&: dstTyAttr);
1904 case FPTruncConvOp::F16x2_TO_F8x2:
1905 return convertFromPacked<NVVM::ConvertF16x2ToF8x2Op>(
1906 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getF16Type(), resultTy: i16Ty, /*sat=*/args: nullptr,
1907 args&: reluAttr, args&: dstTyAttr);
1908 case FPTruncConvOp::F16x2_TO_F6x2:
1909 return convertFromPacked<NVVM::ConvertF16x2ToF6x2Op>(
1910 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getF16Type(), resultTy: i16Ty, args&: reluAttr,
1911 args&: actualDstTyAttr);
1912 case FPTruncConvOp::F16x2_TO_F4x2:
1913 return convertFromPacked<NVVM::ConvertF16x2ToF4x2Op>(
1914 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getF16Type(), resultTy: i8Ty, args&: reluAttr,
1915 args&: actualDstTyAttr);
1916 case FPTruncConvOp::BF16x2_TO_F8x2:
1917 return convertFromPacked<NVVM::ConvertBF16x2ToF8x2Op>(
1918 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getBF16Type(), resultTy: i16Ty, args&: rndAttr, args&: satAttr,
1919 args&: reluAttr, args&: dstTyAttr);
1920 case FPTruncConvOp::BF16x2_TO_F6x2:
1921 return convertFromPacked<NVVM::ConvertBF16x2ToF6x2Op>(
1922 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getBF16Type(), resultTy: i16Ty, args&: reluAttr,
1923 args&: actualDstTyAttr);
1924 case FPTruncConvOp::BF16x2_TO_F4x2:
1925 return convertFromPacked<NVVM::ConvertBF16x2ToF4x2Op>(
1926 b, srcI32Vec, srcBaseIdx, srcElemTy: b.getBF16Type(), resultTy: i8Ty, args&: reluAttr,
1927 args&: actualDstTyAttr);
1928 }
1929 llvm_unreachable("unhandled FPTruncConvOp");
1930}
1931
1932static LogicalResult lowerTruncf(nvgpu::TruncfOp op,
1933 nvgpu::TruncfOp::Adaptor adaptor,
1934 ConversionPatternRewriter &rewriter,
1935 const LLVMTypeConverter *typeConverter) {
1936 MLIRContext *ctx = op.getContext();
1937 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
1938 IntegerType i32Ty = b.getI32Type();
1939 IntegerType i64Ty = b.getI64Type();
1940 static constexpr int regBits = 32;
1941
1942 auto srcType = llvm::dyn_cast<VectorType>(Val: op.getIn().getType());
1943 auto dstType = llvm::dyn_cast<VectorType>(Val: op.getOut().getType());
1944 if (!srcType || srcType.getRank() != 1 || !dstType || dstType.getRank() != 1)
1945 return rewriter.notifyMatchFailure(
1946 arg&: op, msg: "expected 1-D vector; canonicalize pattern handles other shapes");
1947
1948 auto srcElemType = srcType.getElementType();
1949 auto dstElemType = dstType.getElementType();
1950 int srcBW = srcType.getElementTypeBitWidth();
1951 int dstBW = dstType.getElementTypeBitWidth();
1952 int numElems = srcType.getNumElements();
1953
1954 NVVM::FPRoundingModeAttr rndModeAttr = op.getRndAttr();
1955 NVVM::SaturationModeAttr satModeAttr = op.getSatAttr();
1956 auto reluBoolAttr = op.getReluAttr();
1957 Value randomBits = adaptor.getRandomBits();
1958 Type actualDstFloatType = dstElemType;
1959
1960 // STEP 1: bitcast input vector to i32 vector type.
1961 // f64 -> f32/f16/bf16 lowers to a single direct LLVM fptrunc
1962 // f64 -> f8/f6/f4 first truncates to f32 and then reuses the narrow
1963 // conversion path below.
1964 Value input = adaptor.getIn();
1965 if (srcBW == 64) {
1966 if (dstBW >= 16) {
1967 Type convertedType = typeConverter->convertType(t: dstType);
1968 assert(convertedType && "failed to convert type");
1969 Value result = b.create<LLVM::FPTruncOp>(args&: convertedType, args&: input);
1970 rewriter.replaceOp(op, newValues: result);
1971 return success();
1972 }
1973 auto f32VecTy = VectorType::get(shape: srcType.getShape(), elementType: b.getF32Type());
1974 input = b.create<LLVM::FPTruncOp>(args&: f32VecTy, args&: input);
1975 srcType = f32VecTy;
1976 srcElemType = b.getF32Type();
1977 srcBW = 32;
1978 }
1979
1980 // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
1981 int effectiveDstBW = getEffectiveBitWidth(bitWidth: dstBW);
1982
1983 int srcI32Elems = numElems * srcBW / regBits;
1984 int dstI32Elems = numElems * effectiveDstBW / regBits;
1985 Value srcI32Vec =
1986 b.create<LLVM::BitcastOp>(args: VectorType::get(shape: srcI32Elems, elementType: i32Ty), args&: input);
1987 Value dstI32Vec =
1988 b.create<LLVM::UndefOp>(args: VectorType::get(shape: dstI32Elems, elementType: i32Ty));
1989
1990 // STEP 2: look up the conversion op from the (srcType, dstType) table.
1991 auto convEntry = lookupConvOp(table: kFPTruncTable, srcElemType, dstElemType);
1992 if (!convEntry)
1993 return rewriter.notifyMatchFailure(
1994 arg&: op, msg: "unsupported type combination for truncation");
1995 FPTruncConvOp convOp = convEntry->convOp;
1996
1997 // Number of source-side i32 register slots consumed by each NVVM convert Op.
1998 auto getNumSrcI32PerConvert = [](FPKind src) {
1999 return src == FPKind::F32 ? 2 : 1;
2000 };
2001 int numSrcI32PerConv = getNumSrcI32PerConvert(convEntry->src);
2002
2003 // STEP 3: pack conversion results into destination i32 vector.
2004 const int srcStep = srcBW / effectiveDstBW;
2005 const int resultBW =
2006 effectiveDstBW * 2; // each conversion produces 2 (packed) elements
2007 const int numConvsPerI32 = regBits / resultBW;
2008
2009 for (int srcIdx = 0, dstIdx = 0; dstIdx < dstI32Elems;
2010 srcIdx += srcStep, dstIdx++) {
2011 Value dstIdxConst =
2012 b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: dstIdx));
2013 Value dstValue;
2014
2015 if (numConvsPerI32 == 1) {
2016 // f16/bf16 destinations
2017 dstValue = createTruncConversion(
2018 b, ctx, convOp, srcI32Vec, srcBaseIdx: srcIdx, rndAttr: rndModeAttr, satAttr: satModeAttr,
2019 reluAttr: reluBoolAttr, dstElemType, actualDstFloatType, randomBits);
2020 } else {
2021 // f8/f6/f4 destinations: pack sub-results via vector insert + bitcast.
2022 auto subResultType = IntegerType::get(context: ctx, width: resultBW);
2023 auto subVecTy = VectorType::get(shape: numConvsPerI32, elementType: subResultType);
2024 Value subVec = b.create<LLVM::UndefOp>(args&: subVecTy);
2025
2026 int insertIdx = numConvsPerI32 - 1;
2027 int curStep = srcStep;
2028 while (curStep > 0) {
2029 curStep -= numSrcI32PerConv;
2030 Value subResult = createTruncConversion(
2031 b, ctx, convOp, srcI32Vec, srcBaseIdx: srcIdx + curStep, rndAttr: rndModeAttr,
2032 satAttr: satModeAttr, reluAttr: reluBoolAttr, dstElemType, actualDstFloatType,
2033 /*randomBits=*/Value());
2034 subVec = b.create<LLVM::InsertElementOp>(
2035 args&: subVec, args&: subResult,
2036 args: b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: insertIdx)));
2037 insertIdx--;
2038 }
2039
2040 dstValue = b.create<LLVM::BitcastOp>(args&: i32Ty, args&: subVec);
2041 }
2042
2043 dstI32Vec =
2044 b.create<LLVM::InsertElementOp>(args&: dstI32Vec, args&: dstValue, args&: dstIdxConst);
2045 }
2046
2047 // STEP 4: produce final result.
2048 Type convertedType = typeConverter->convertType(t: dstType);
2049 assert(convertedType && "failed to convert type");
2050 if (convEntry->dst == FPKind::F6) {
2051 IntegerType i8Ty = b.getI8Type();
2052 auto i8VecTy = VectorType::get(shape: numElems, elementType: i8Ty);
2053 Value i8Vec = b.create<LLVM::BitcastOp>(args&: i8VecTy, args&: dstI32Vec);
2054 Value truncVec = b.create<LLVM::TruncOp>(args&: convertedType, args&: i8Vec);
2055 rewriter.replaceOp(op, newValues: truncVec);
2056 } else {
2057 auto dstVec = b.create<LLVM::BitcastOp>(args&: convertedType, args&: dstI32Vec);
2058 rewriter.replaceOp(op, newOp: dstVec);
2059 }
2060 return success();
2061}
2062
2063struct NVGPUTruncfOpLowering : public ConvertOpToLLVMPattern<nvgpu::TruncfOp> {
2064 using ConvertOpToLLVMPattern<nvgpu::TruncfOp>::ConvertOpToLLVMPattern;
2065
2066 LogicalResult
2067 matchAndRewrite(nvgpu::TruncfOp op, OpAdaptor adaptor,
2068 ConversionPatternRewriter &rewriter) const override {
2069 return lowerTruncf(op, adaptor, rewriter, typeConverter: getTypeConverter());
2070 }
2071};
2072
2073//===----------------------------------------------------------------------===//
2074// NVGPUExtfOp Lowering
2075//===----------------------------------------------------------------------===//
2076
2077/// Conversion op identifier for nvgpu.extf lowering dispatch table.
2078enum class FPExtConvOp {
2079 F8x2_TO_F16x2,
2080 F8x2_TO_BF16x2,
2081 F6x2_TO_F16x2,
2082 F6x2_TO_BF16x2,
2083 F4x2_TO_F16x2,
2084 F4x2_TO_BF16x2,
2085};
2086
2087struct FPExtTableEntry {
2088 FPKind src;
2089 FPKind dst;
2090 FPExtConvOp convOp;
2091};
2092
2093static constexpr FPExtTableEntry kFPExtTable[] = {
2094 {.src: FPKind::F8, .dst: FPKind::F16, .convOp: FPExtConvOp::F8x2_TO_F16x2},
2095 {.src: FPKind::F8, .dst: FPKind::BF16, .convOp: FPExtConvOp::F8x2_TO_BF16x2},
2096 {.src: FPKind::F6, .dst: FPKind::F16, .convOp: FPExtConvOp::F6x2_TO_F16x2},
2097 {.src: FPKind::F6, .dst: FPKind::BF16, .convOp: FPExtConvOp::F6x2_TO_BF16x2},
2098 {.src: FPKind::F4, .dst: FPKind::F16, .convOp: FPExtConvOp::F4x2_TO_F16x2},
2099 {.src: FPKind::F4, .dst: FPKind::BF16, .convOp: FPExtConvOp::F4x2_TO_BF16x2},
2100};
2101
2102/// Create a typed NVVM extension conversion.
2103/// For f8/f6: src is vector<2xi8>. For f4: src is i8.
2104/// Returns i32 (bitcast from vector<2xf16> or vector<2xbf16>).
2105static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
2106 FPExtConvOp convOp, Value src,
2107 BoolAttr reluAttr, Type actualSrcFloatType,
2108 Value extScaleFactor = Value()) {
2109 IntegerType i32Ty = b.getI32Type();
2110 auto srcTyAttr = TypeAttr::get(type: actualSrcFloatType);
2111
2112 switch (convOp) {
2113 case FPExtConvOp::F8x2_TO_F16x2: {
2114 Value r = NVVM::ConvertF8x2ToF16x2Op::create(
2115 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getF16Type()), src, srcType: srcTyAttr, relu: reluAttr);
2116 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2117 }
2118 case FPExtConvOp::F8x2_TO_BF16x2: {
2119 Value r = NVVM::ConvertF8x2ToBF16x2Op::create(
2120 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getBF16Type()), src, scaleFactor: extScaleFactor, srcType: srcTyAttr);
2121 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2122 }
2123 case FPExtConvOp::F6x2_TO_F16x2: {
2124 Value r = NVVM::ConvertF6x2ToF16x2Op::create(
2125 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getF16Type()), src, srcType: srcTyAttr, relu: reluAttr);
2126 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2127 }
2128 case FPExtConvOp::F6x2_TO_BF16x2: {
2129 Value r = NVVM::ConvertF6x2ToBF16x2Op::create(
2130 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getBF16Type()), src, scaleFactor: extScaleFactor, srcType: srcTyAttr);
2131 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2132 }
2133 case FPExtConvOp::F4x2_TO_F16x2: {
2134 Value r = NVVM::ConvertF4x2ToF16x2Op::create(
2135 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getF16Type()), src, srcType: srcTyAttr, relu: reluAttr);
2136 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2137 }
2138 case FPExtConvOp::F4x2_TO_BF16x2: {
2139 Value r = NVVM::ConvertF4x2ToBF16x2Op::create(
2140 builder&: b, dst: VectorType::get(shape: 2, elementType: b.getBF16Type()), src, scaleFactor: extScaleFactor, srcType: srcTyAttr);
2141 return b.create<LLVM::BitcastOp>(args&: i32Ty, args&: r);
2142 }
2143 }
2144 llvm_unreachable("unhandled FPExtConvOp");
2145}
2146
2147static LogicalResult lowerExtf(nvgpu::ExtfOp op, nvgpu::ExtfOp::Adaptor adaptor,
2148 ConversionPatternRewriter &rewriter,
2149 const LLVMTypeConverter *typeConverter) {
2150 MLIRContext *ctx = op.getContext();
2151 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
2152 IntegerType i8Ty = b.getI8Type();
2153 IntegerType i16Ty = b.getI16Type();
2154 IntegerType i32Ty = b.getI32Type();
2155 IntegerType i64Ty = b.getI64Type();
2156
2157 static constexpr int regBits = 32;
2158 auto srcType = llvm::dyn_cast<VectorType>(Val: op.getIn().getType());
2159 auto dstType = llvm::dyn_cast<VectorType>(Val: op.getOut().getType());
2160 if (!srcType || srcType.getRank() != 1 || !dstType || dstType.getRank() != 1)
2161 return rewriter.notifyMatchFailure(
2162 arg&: op, msg: "expected 1-D vector; canonicalize pattern handles other shapes");
2163
2164 auto srcElemType = srcType.getElementType();
2165 auto dstElemType = dstType.getElementType();
2166 int srcBW = srcType.getElementTypeBitWidth();
2167 int dstBW = dstType.getElementTypeBitWidth();
2168 int numElems = srcType.getNumElements();
2169
2170 auto reluBoolAttr = op.getReluAttr();
2171 Type actualSrcFloatType = srcElemType;
2172
2173 assert(dstBW == 16 || dstBW == 32 || dstBW == 64);
2174
2175 // Wide source (f16/bf16/f32) to wide destination (f32/f64): single FPExt.
2176 if (srcBW >= 16 && dstBW >= 32) {
2177 Value result = adaptor.getIn();
2178 if (srcElemType != dstElemType) {
2179 Type convertedType = typeConverter->convertType(t: dstType);
2180 assert(convertedType && "failed to convert type");
2181 result = b.create<LLVM::FPExtOp>(args&: convertedType, args&: result);
2182 }
2183 rewriter.replaceOp(op, newValues: result);
2184 return success();
2185 }
2186
2187 // Narrow source (f8/f6/f4): NVVM typed op produces f16/bf16; optionally
2188 // followed by FPExt to the final f32/f64 destination.
2189 bool needsFinalFPExt = (dstBW >= 32);
2190 Type intermediateDstElem = dstElemType;
2191 if (needsFinalFPExt && llvm::isa<Float8E8M0FNUType>(Val: srcElemType))
2192 intermediateDstElem = b.getBF16Type();
2193 else if (needsFinalFPExt)
2194 intermediateDstElem = b.getF16Type();
2195 int intermediateDstBW = needsFinalFPExt ? 16 : dstBW;
2196
2197 // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
2198 int effectiveSrcBW = getEffectiveBitWidth(bitWidth: srcBW);
2199
2200 // STEP 1: prepare input as i32 register vector.
2201 // For f6: zext from vector<Nxi6> to vector<Nxi8>, then bitcast to i32s.
2202 Value inputVec = adaptor.getIn();
2203 if (srcBW == 6) {
2204 auto i8VecTy = VectorType::get(shape: numElems, elementType: i8Ty);
2205 inputVec = b.create<LLVM::ZExtOp>(args&: i8VecTy, args&: inputVec);
2206 }
2207
2208 int srcI32Elems = numElems * effectiveSrcBW / regBits;
2209 int dstI32Elems = numElems * intermediateDstBW / regBits;
2210 Value srcI32Vec =
2211 b.create<LLVM::BitcastOp>(args: VectorType::get(shape: srcI32Elems, elementType: i32Ty), args&: inputVec);
2212 Value dstI32Vec =
2213 b.create<LLVM::UndefOp>(args: VectorType::get(shape: dstI32Elems, elementType: i32Ty));
2214
2215 // STEP 2: look up the conversion op from the (srcType, dstType) table.
2216 auto convEntry = lookupConvOp(table: kFPExtTable, srcElemType, dstElemType: intermediateDstElem);
2217 if (!convEntry)
2218 return rewriter.notifyMatchFailure(
2219 arg&: op, msg: "unsupported type combination for extension");
2220 FPExtConvOp convOp = convEntry->convOp;
2221 Value extScaleFactor;
2222
2223 // STEP 3: iterate over source i32 elements, producing destination i32s.
2224 for (int srcIdx = 0, dstIdx = 0; srcIdx < srcI32Elems; srcIdx++) {
2225 Value srcI32 = b.create<LLVM::ExtractElementOp>(
2226 args&: srcI32Vec,
2227 args: b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: srcIdx)));
2228
2229 if (effectiveSrcBW == 8) {
2230 // f8/f6: one i32 holds 4 bytes -> split into 2 pairs of i16 -> 2 convs.
2231 Value i16Vec =
2232 b.create<LLVM::BitcastOp>(args: VectorType::get(shape: 2, elementType: i16Ty), args&: srcI32);
2233 for (int half = 0; half < 2; half++) {
2234 Value halfI16 = b.create<LLVM::ExtractElementOp>(
2235 args&: i16Vec,
2236 args: b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: half)));
2237 Value src =
2238 b.create<LLVM::BitcastOp>(args: VectorType::get(shape: 2, elementType: i8Ty), args&: halfI16);
2239 Value dstValue =
2240 createExtConversion(b, ctx, convOp, src, reluAttr: reluBoolAttr,
2241 actualSrcFloatType, extScaleFactor);
2242 Value dstIdxConst =
2243 b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: dstIdx));
2244 dstI32Vec =
2245 b.create<LLVM::InsertElementOp>(args&: dstI32Vec, args&: dstValue, args&: dstIdxConst);
2246 dstIdx++;
2247 }
2248 } else {
2249 // f4: one i32 holds 4 bytes -> each byte is one conversion input.
2250 Value i8Vec = b.create<LLVM::BitcastOp>(args: VectorType::get(shape: 4, elementType: i8Ty), args&: srcI32);
2251 for (int byteIdx = 0; byteIdx < 4; byteIdx++) {
2252 Value src = b.create<LLVM::ExtractElementOp>(
2253 args&: i8Vec,
2254 args: b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: byteIdx)));
2255 Value dstValue =
2256 createExtConversion(b, ctx, convOp, src, reluAttr: reluBoolAttr,
2257 actualSrcFloatType, extScaleFactor);
2258 Value dstIdxConst =
2259 b.create<LLVM::ConstantOp>(args&: i64Ty, args: b.getI64IntegerAttr(value: dstIdx));
2260 dstI32Vec =
2261 b.create<LLVM::InsertElementOp>(args&: dstI32Vec, args&: dstValue, args&: dstIdxConst);
2262 dstIdx++;
2263 }
2264 }
2265 }
2266
2267 // STEP 4: produce final result.
2268 Type convertedType = typeConverter->convertType(t: dstType);
2269 assert(convertedType && "failed to convert type");
2270 Value result;
2271 if (needsFinalFPExt) {
2272 auto intermediateVecTy = VectorType::get(shape: numElems, elementType: intermediateDstElem);
2273 Value intermediateVec =
2274 b.create<LLVM::BitcastOp>(args&: intermediateVecTy, args&: dstI32Vec);
2275 result = b.create<LLVM::FPExtOp>(args&: convertedType, args&: intermediateVec);
2276 } else {
2277 result = b.create<LLVM::BitcastOp>(args&: convertedType, args&: dstI32Vec);
2278 }
2279 rewriter.replaceOp(op, newValues: result);
2280 return success();
2281}
2282
2283struct NVGPUExtfOpLowering : public ConvertOpToLLVMPattern<nvgpu::ExtfOp> {
2284 using ConvertOpToLLVMPattern<nvgpu::ExtfOp>::ConvertOpToLLVMPattern;
2285
2286 LogicalResult
2287 matchAndRewrite(nvgpu::ExtfOp op, OpAdaptor adaptor,
2288 ConversionPatternRewriter &rewriter) const override {
2289 return lowerExtf(op, adaptor, rewriter, typeConverter: getTypeConverter());
2290 }
2291};
2292
2293static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
2294 int step) {
2295 static constexpr int regBits = 32;
2296 int effSrcBW = getEffectiveBitWidth(bitWidth: srcBW);
2297 int effDstBW = getEffectiveBitWidth(bitWidth: dstBW);
2298 auto ceilDiv = [](int64_t x, int64_t y) { return (x + y - 1) / y; };
2299 int64_t padded =
2300 std::max(a: ceilDiv(numElems * effSrcBW, regBits) * regBits / effSrcBW,
2301 b: ceilDiv(numElems * effDstBW, regBits) * regBits / effDstBW);
2302 return ceilDiv(padded, step) * step;
2303}
2304
2305/// Canonicalization pattern for nvgpu.truncf / nvgpu.extf:
2306/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
2307/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
2308template <typename CvtOp, bool IsTrunc>
2309struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
2310 using OpRewritePattern<CvtOp>::OpRewritePattern;
2311
2312 LogicalResult matchAndRewrite(CvtOp op,
2313 PatternRewriter &rewriter) const override {
2314 Type inType = op.getIn().getType();
2315 Type outType = op.getOut().getType();
2316
2317 Type srcElemTy = getElementTypeOrSelf(type: inType);
2318 Type dstElemTy = getElementTypeOrSelf(type: outType);
2319 int srcBW = srcElemTy.getIntOrFloatBitWidth();
2320 int dstBW = dstElemTy.getIntOrFloatBitWidth();
2321 int effSrcBW = getEffectiveBitWidth(bitWidth: srcBW);
2322 int effDstBW = getEffectiveBitWidth(bitWidth: dstBW);
2323
2324 bool isScalar = !isa<VectorType>(Val: inType);
2325 auto srcVecTy = dyn_cast<VectorType>(Val&: inType);
2326 bool isMultiRank = srcVecTy && srcVecTy.getRank() > 1;
2327 int64_t numElems = isScalar ? 1 : srcVecTy.getNumElements();
2328 int step = IsTrunc ? effSrcBW / effDstBW : effDstBW / effSrcBW;
2329 int64_t paddedElems = computePaddedElems(numElems, srcBW, dstBW, step);
2330 bool needsPad = (paddedElems != numElems);
2331
2332 if (!isScalar && !isMultiRank && !needsPad)
2333 return failure();
2334
2335 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
2336 Value input = op.getIn();
2337
2338 if (isScalar)
2339 input = vector::BroadcastOp::create(builder&: b, result: VectorType::get(shape: {1}, elementType: srcElemTy),
2340 source: input);
2341 if (isMultiRank)
2342 input = vector::ShapeCastOp::create(
2343 builder&: b, result: VectorType::get(shape: {numElems}, elementType: srcElemTy), source: input);
2344
2345 if (needsPad) {
2346 auto paddedTy = VectorType::get(shape: {paddedElems}, elementType: srcElemTy);
2347 Value zero = arith::ConstantOp::create(
2348 builder&: b, value: DenseElementsAttr::get(type: paddedTy, values: b.getZeroAttr(type: srcElemTy)));
2349 input = vector::InsertStridedSliceOp::create(
2350 builder&: b, valueToStore: input, dest: zero, offsets: SmallVector<int64_t>{0}, strides: SmallVector<int64_t>{1});
2351 }
2352
2353 auto cvtDstTy =
2354 VectorType::get(shape: {needsPad ? paddedElems : numElems}, elementType: dstElemTy);
2355 Value cvt;
2356 if constexpr (IsTrunc) {
2357 cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(),
2358 op.getReluAttr(), op.getRandomBits());
2359 } else {
2360 cvt =
2361 CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getReluAttr());
2362 }
2363 Value result = cvt;
2364
2365 if (needsPad) {
2366 result = vector::ExtractStridedSliceOp::create(
2367 builder&: b, source: result, offsets: SmallVector<int64_t>{0}, sizes: SmallVector<int64_t>{numElems},
2368 strides: SmallVector<int64_t>{1});
2369 }
2370
2371 if (isMultiRank) {
2372 result =
2373 vector::ShapeCastOp::create(builder&: b, result: cast<VectorType>(Val&: outType), source: result);
2374 }
2375
2376 if (isScalar) {
2377 result = vector::ExtractOp::create(builder&: b, source: result, position: SmallVector<int64_t>{0});
2378 }
2379
2380 rewriter.replaceOp(op, result);
2381 return success();
2382 }
2383};
2384
2385using NVGPUTruncfCanonicalizePattern =
2386 NVGPUFPCanonicalizePattern<nvgpu::TruncfOp, true>;
2387using NVGPUExtfCanonicalizePattern =
2388 NVGPUFPCanonicalizePattern<nvgpu::ExtfOp, false>;
2389} // namespace
2390
2391void mlir::nvgpu::populateCommonGPUTypeAndAttributeConversions(
2392 TypeConverter &typeConverter) {
2393 // NVVM uses alloca in the default address space to represent private
2394 // memory allocations, so drop private annotations. NVVM uses address
2395 // space 3 for shared memory. NVVM uses the default address space to
2396 // represent global memory.
2397 populateGpuMemorySpaceAttributeConversions(
2398 typeConverter, mapping: [](gpu::AddressSpace space) -> unsigned {
2399 switch (space) {
2400 case gpu::AddressSpace::Global:
2401 return static_cast<unsigned>(NVVM::NVVMMemorySpace::Global);
2402 case gpu::AddressSpace::Workgroup:
2403 return static_cast<unsigned>(NVVM::NVVMMemorySpace::Shared);
2404 case gpu::AddressSpace::Private:
2405 return 0;
2406 case gpu::AddressSpace::Constant:
2407 return static_cast<unsigned>(NVVM::NVVMMemorySpace::Constant);
2408 }
2409 llvm_unreachable("unknown address space enum value");
2410 });
2411}
2412
2413void mlir::populateNVGPUToNVVMConversionPatterns(
2414 const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
2415 patterns.add<
2416 NVGPUMBarrierCreateLowering, // nvgpu.mbarrier.create
2417 NVGPUMBarrierInitLowering, // nvgpu.mbarrier.init
2418 NVGPUMBarrierGetLowering, // nvgpu.mbarrier.get
2419 NVGPUMBarrierArriveLowering, // nvgpu.mbarrier.arrive
2420 NVGPUMBarrierArriveNoCompleteLowering, // nvgpu.mbarrier.arrive.no_complete
2421 NVGPUMBarrierTestWaitLowering, // nvgpu.mbarrier.test_wait_parity
2422 NVGPUMBarrierTryWaitParityLowering, // nvgpu.mbarrier.try_wait_parity
2423 NVGPUTmaAsyncLoadOpLowering, // nvgpu.tma.async.load
2424 NVGPUTmaAsyncStoreOpLowering, // nvgpu.tma.async.store
2425 NVGPUTmaCreateDescriptorOpLowering, // nvgpu.tma.create.descriptor
2426 NVGPUTmaPrefetchOpLowering, // nvgpu.tma.prefetch.descriptor
2427 NVGPUTmaFenceOpLowering, // nvgpu.tma.fence.descriptor
2428 NVGPUMBarrierArriveExpectTxLowering, // nvgpu.mbarrier.arrive.expect_tx
2429 NVGPUGenerateWarpgroupDescriptorLowering, // nvgpu.warpgroup.generate.descriptor
2430 NVGPUWarpgroupMmaOpLowering, // nvgpu.warpgroup.mma
2431 NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
2432 NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
2433 NVGPUTruncfOpLowering, // nvgpu.truncf
2434 NVGPUExtfOpLowering, // nvgpu.extf
2435 MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
2436 NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
2437 NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(arg: converter);
2438
2439 patterns.add<NVGPUTruncfCanonicalizePattern, NVGPUExtfCanonicalizePattern>(
2440 arg: patterns.getContext());
2441}
2442