1//===------ WmmaOpsToNVVM.cpp - WMMA LD/ST/Compute to NVVM lowering -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains definitions of patterns to lower GPU Subgroup MMA ops to
10// NVVM Dialect.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
15#include "mlir/Conversion/LLVMCommon/Pattern.h"
16#include "mlir/Dialect/GPU/IR/GPUDialect.h"
17#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
21#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
22#include "mlir/IR/TypeUtilities.h"
23#include "mlir/IR/Types.h"
24
25using namespace mlir;
26
27namespace {
28
29/// Checks if all the operands of the op being lowered are of LLVM Types. The
30/// types are expected to be converted by the `LLVMTypeConverter` before the op
31/// is actually lowered. If the type of an operands is not already converted it
32/// hints a missing typeConversion and failure is returned in that case.
33static LogicalResult areAllLLVMTypes(Operation *op, ValueRange operands,
34 ConversionPatternRewriter &rewriter) {
35 if (!llvm::all_of(Range&: operands, P: [](Value value) {
36 return LLVM::isCompatibleType(type: value.getType());
37 })) {
38 return rewriter.notifyMatchFailure(
39 arg&: op, msg: "cannot convert if operands aren't of LLVM type.");
40 }
41
42 return success();
43}
44
45/// Error string to emit when an unimplemented WMMA variant is encountered.
46static constexpr StringRef kInvalidCaseStr = "Unsupported WMMA variant.";
47
48static NVVM::MMAFrag convertOperand(StringRef operandName) {
49 if (operandName == "AOp")
50 return NVVM::MMAFrag::a;
51 if (operandName == "BOp")
52 return NVVM::MMAFrag::b;
53 if (operandName == "COp")
54 return NVVM::MMAFrag::c;
55 llvm_unreachable("Unknown operand name");
56}
57
58static NVVM::MMATypes getElementType(gpu::MMAMatrixType type) {
59 if (type.getElementType().isF16())
60 return NVVM::MMATypes::f16;
61 if (type.getElementType().isF32())
62 return type.getOperand() == "COp" ? NVVM::MMATypes::f32
63 : NVVM::MMATypes::tf32;
64 if (type.getElementType().isF64())
65 return NVVM::MMATypes::f64;
66 if (type.getElementType().isSignedInteger(width: 8))
67 return NVVM::MMATypes::s8;
68 if (type.getElementType().isUnsignedInteger(width: 8))
69 return NVVM::MMATypes::u8;
70 // Accumulator type is signless and implies signed.
71 if (type.getElementType().isInteger(width: 32))
72 return NVVM::MMATypes::s32;
73 llvm_unreachable("Unsupported type");
74}
75
76/// This class implements the conversion of GPU MMA loadOp to wmma.load op
77/// in the NVVM dialect. The conversion not only emits the NVVM op but also
78/// emits code that is necessary to store the data in the destination memref
79/// after it has been loaded.
80struct WmmaLoadOpToNVVMLowering
81 : public ConvertOpToLLVMPattern<gpu::SubgroupMmaLoadMatrixOp> {
82 using ConvertOpToLLVMPattern<
83 gpu::SubgroupMmaLoadMatrixOp>::ConvertOpToLLVMPattern;
84
85 LogicalResult
86 matchAndRewrite(gpu::SubgroupMmaLoadMatrixOp subgroupMmaLoadMatrixOp,
87 OpAdaptor adaptor,
88 ConversionPatternRewriter &rewriter) const override {
89 Operation *op = subgroupMmaLoadMatrixOp.getOperation();
90 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)))
91 return failure();
92
93 // Get the shape of the MMAMatrix type being returned. The shape will
94 // choose which intrinsic this op will be lowered to.
95 NVVM::MMALayout layout = subgroupMmaLoadMatrixOp.getTranspose()
96 ? NVVM::MMALayout::col
97 : NVVM::MMALayout::row;
98 gpu::MMAMatrixType retType =
99 cast<gpu::MMAMatrixType>(Val: subgroupMmaLoadMatrixOp.getRes().getType());
100 ArrayRef<int64_t> retTypeShape = retType.getShape();
101 int64_t m = 0;
102 int64_t n = 0;
103 int64_t k = 0;
104 NVVM::MMATypes eltype = getElementType(type: retType);
105 // NVVM intrinsics require to give mxnxk dimensions, infer the missing
106 // dimension based on the valid intrinsics available.
107 if (retType.getOperand() == "AOp") {
108 m = retTypeShape[0];
109 k = retTypeShape[1];
110 n = NVVM::WMMALoadOp::inferNDimension(m, k, eltypeEnum: eltype);
111 } else if (retType.getOperand() == "BOp") {
112 k = retTypeShape[0];
113 n = retTypeShape[1];
114 m = NVVM::WMMALoadOp::inferMDimension(k, n, eltypeEnum: eltype);
115 } else if (retType.getOperand() == "COp") {
116 m = retTypeShape[0];
117 n = retTypeShape[1];
118 k = NVVM::WMMALoadOp::inferKDimension(m, n, eltypeEnum: eltype);
119 }
120 NVVM::MMAFrag frag = convertOperand(operandName: retType.getOperand());
121 // Check that there is an exisiting instruction for the combination we need.
122 if (NVVM::WMMALoadOp::getIntrinsicID(m, n, k, layoutEnum: layout, eltypeEnum: eltype, fragEnum: frag) == 0)
123 return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr);
124
125 Type resType = convertMMAToLLVMType(type: retType);
126 Location loc = op->getLoc();
127
128 // Create nvvm.mma_load op according to the operand types.
129 Value dataPtr = getStridedElementPtr(
130 rewriter, loc,
131 type: cast<MemRefType>(Val: subgroupMmaLoadMatrixOp.getSrcMemref().getType()),
132 memRefDesc: adaptor.getSrcMemref(), indices: adaptor.getIndices());
133
134 // The NVVM op takes the leading dimension as an `i32`. Reject a value that
135 // does not fit rather than silently truncating it.
136 int64_t leadDimension =
137 subgroupMmaLoadMatrixOp.getLeadDimension().getSExtValue();
138 if (!llvm::isInt<32>(x: leadDimension))
139 return rewriter.notifyMatchFailure(
140 arg&: op, msg: "leading dimension does not fit into an i32");
141 Value leadingDim = LLVM::ConstantOp::create(
142 builder&: rewriter, location: loc, res: rewriter.getI32Type(),
143 value: rewriter.getI32IntegerAttr(value: static_cast<int32_t>(leadDimension)));
144 rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>(
145 op, args&: resType, args&: dataPtr, args&: leadingDim, args&: m, args&: n, args&: k, args&: layout, args&: eltype, args&: frag);
146 return success();
147 }
148};
149
150/// This class implements the conversion of GPU MMA storeOp to wmma.store op
151/// in the NVVM dialect. The conversion not only emits the NVVM op but also
152/// emits code that is necessary to unpack the data in the source and
153/// convert the data in the format that is needed by the NVVM op.
154struct WmmaStoreOpToNVVMLowering
155 : public ConvertOpToLLVMPattern<gpu::SubgroupMmaStoreMatrixOp> {
156 using ConvertOpToLLVMPattern<
157 gpu::SubgroupMmaStoreMatrixOp>::ConvertOpToLLVMPattern;
158
159 LogicalResult
160 matchAndRewrite(gpu::SubgroupMmaStoreMatrixOp subgroupMmaStoreMatrixOp,
161 OpAdaptor adaptor,
162 ConversionPatternRewriter &rewriter) const override {
163 Operation *op = subgroupMmaStoreMatrixOp.getOperation();
164 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)))
165 return failure();
166
167 Location loc = op->getLoc();
168
169 SmallVector<Value, 4> storeOpOperands;
170 // Get the shape of the MMAMatrix type being stored. The shape will
171 // choose which intrinsic this op will be lowered to.
172 gpu::MMAMatrixType srcType =
173 cast<gpu::MMAMatrixType>(Val: subgroupMmaStoreMatrixOp.getSrc().getType());
174 ArrayRef<int64_t> srcTypeShape = srcType.getShape();
175 NVVM::MMALayout layout = subgroupMmaStoreMatrixOp.getTranspose()
176 ? NVVM::MMALayout::col
177 : NVVM::MMALayout::row;
178 NVVM::MMATypes eltype = getElementType(type: srcType);
179 int64_t m = srcTypeShape[0];
180 int64_t n = srcTypeShape[1];
181 int64_t k = NVVM::WMMAStoreOp::inferKDimension(m, n, eltypeEnum: eltype);
182 if (NVVM::WMMAStoreOp::getIntrinsicID(m, n, k, layoutEnum: layout, eltypeEnum: eltype) == 0)
183 return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr);
184
185 auto matrixType = cast<LLVM::LLVMStructType>(Val: adaptor.getSrc().getType());
186 for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) {
187 Value toUse =
188 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: adaptor.getSrc(), position: i);
189 storeOpOperands.push_back(Elt: toUse);
190 }
191
192 Value dataPtr = getStridedElementPtr(
193 rewriter, loc,
194 type: cast<MemRefType>(Val: subgroupMmaStoreMatrixOp.getDstMemref().getType()),
195 memRefDesc: adaptor.getDstMemref(), indices: adaptor.getIndices());
196 // The NVVM op takes the leading dimension as an `i32`. Reject a value that
197 // does not fit rather than silently truncating it.
198 int64_t leadDimension =
199 subgroupMmaStoreMatrixOp.getLeadDimension().getSExtValue();
200 if (!llvm::isInt<32>(x: leadDimension))
201 return rewriter.notifyMatchFailure(
202 arg&: op, msg: "leading dimension does not fit into an i32");
203 Value leadingDim = LLVM::ConstantOp::create(
204 builder&: rewriter, location: loc, res: rewriter.getI32Type(),
205 value: rewriter.getI32IntegerAttr(value: static_cast<int32_t>(leadDimension)));
206 rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>(
207 op, args&: dataPtr, args&: m, args&: n, args&: k, args&: layout, args&: eltype, args&: storeOpOperands, args&: leadingDim);
208 return success();
209 }
210};
211
212/// This class implements the conversion of GPU MMA computeOp to wmma.mma op
213/// in the NVVM dialect.
214struct WmmaMmaOpToNVVMLowering
215 : public ConvertOpToLLVMPattern<gpu::SubgroupMmaComputeOp> {
216 using ConvertOpToLLVMPattern<
217 gpu::SubgroupMmaComputeOp>::ConvertOpToLLVMPattern;
218
219 LogicalResult
220 matchAndRewrite(gpu::SubgroupMmaComputeOp subgroupMmaComputeOp,
221 OpAdaptor adaptor,
222 ConversionPatternRewriter &rewriter) const override {
223 Operation *op = subgroupMmaComputeOp.getOperation();
224 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)))
225 return failure();
226
227 Location loc = op->getLoc();
228
229 // The wmma.mma intrinsic in llvm requires the operands as individual
230 // values. So individual elements from the memrefs need to be extracted and
231 // then passed on to the intrinsic call. Emit llvm ops to extract individual
232 // values form lowered memrefs.
233 SmallVector<Value> unpackedOps;
234 auto unpackOp = [&](Value operand) {
235 // f64 a and b fragments are not structs but scalars.
236 if (!isa<LLVM::LLVMStructType>(Val: operand.getType())) {
237 unpackedOps.push_back(Elt: operand);
238 return;
239 }
240 // every other type is lowered to an LLVM struct, extract the values.
241 auto structType = cast<LLVM::LLVMStructType>(Val: operand.getType());
242 for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) {
243 Value toUse = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: operand, position: i);
244 unpackedOps.push_back(Elt: toUse);
245 }
246 };
247
248 // Get the shapes of the MMAMatrix type being used. The shapes will
249 // choose which intrinsic this op will be lowered to.
250 gpu::MMAMatrixType aType =
251 cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpA().getType());
252 ArrayRef<int64_t> aTypeShape = aType.getShape();
253 gpu::MMAMatrixType cType =
254 cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpC().getType());
255 ArrayRef<int64_t> cTypeShape = cType.getShape();
256 int64_t m = cTypeShape[0];
257 int64_t n = cTypeShape[1];
258 int64_t k = aTypeShape[1];
259 NVVM::MMALayout aLayout = subgroupMmaComputeOp.getATranspose()
260 ? NVVM::MMALayout::col
261 : NVVM::MMALayout::row;
262 NVVM::MMALayout bLayout = subgroupMmaComputeOp.getBTranspose()
263 ? NVVM::MMALayout::col
264 : NVVM::MMALayout::row;
265 NVVM::MMATypes sourceType = getElementType(type: aType);
266 NVVM::MMATypes destType = getElementType(type: cType);
267 if (NVVM::WMMAMmaOp::getIntrinsicID(m, n, k, layoutAEnum: aLayout, layoutBEnum: bLayout, eltypeAEnum: sourceType,
268 eltypeBEnum: destType) == 0)
269 return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr);
270
271 NVVM::MMATypes bElementType = getElementType(
272 type: cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpB().getType()));
273 if (bElementType != sourceType)
274 return rewriter.notifyMatchFailure(
275 arg&: op, msg: "WMMA compute op input matrix element types must match.");
276
277 unpackOp(adaptor.getOpA());
278 unpackOp(adaptor.getOpB());
279 unpackOp(adaptor.getOpC());
280
281 rewriter.replaceOpWithNewOp<NVVM::WMMAMmaOp>(
282 op, args: adaptor.getOpC().getType(), args&: m, args&: n, args&: k, args&: aLayout, args&: bLayout, args&: sourceType,
283 args&: destType, args&: unpackedOps);
284 return success();
285 }
286};
287
288/// Convert GPU MMA ConstantMatrixOp to a chain of InsertValueOp.
289struct WmmaConstantOpToNVVMLowering
290 : public ConvertOpToLLVMPattern<gpu::SubgroupMmaConstantMatrixOp> {
291 using ConvertOpToLLVMPattern<
292 gpu::SubgroupMmaConstantMatrixOp>::ConvertOpToLLVMPattern;
293
294 LogicalResult
295 matchAndRewrite(gpu::SubgroupMmaConstantMatrixOp subgroupMmaConstantOp,
296 OpAdaptor adaptor,
297 ConversionPatternRewriter &rewriter) const override {
298 if (failed(Result: areAllLLVMTypes(op: subgroupMmaConstantOp.getOperation(),
299 operands: adaptor.getOperands(), rewriter)))
300 return failure();
301 Location loc = subgroupMmaConstantOp.getLoc();
302 Value cst = adaptor.getOperands()[0];
303 Type type = convertMMAToLLVMType(
304 type: cast<gpu::MMAMatrixType>(Val: subgroupMmaConstantOp.getType()));
305 // If the element is not a struct, it means it's a scalar f64.
306 auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: type);
307 if (!structType) {
308 rewriter.replaceOp(op: subgroupMmaConstantOp, newValues: cst);
309 return success();
310 }
311 // If the element type is a vector create a vector from the operand.
312 if (auto vecType = dyn_cast<VectorType>(Val: structType.getBody()[0])) {
313 Value vecCst = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: vecType);
314 for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) {
315 Value idx = LLVM::ConstantOp::create(builder&: rewriter, location: loc,
316 type: rewriter.getI32Type(), value: vecEl);
317 vecCst = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, res: vecType, vector: vecCst,
318 value: cst, position: idx);
319 }
320 cst = vecCst;
321 }
322 Value matrixStruct = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: structType);
323 for (size_t i : llvm::seq(Begin: size_t(0), End: structType.getBody().size())) {
324 matrixStruct =
325 LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: matrixStruct, value: cst, position: i);
326 }
327 rewriter.replaceOp(op: subgroupMmaConstantOp, newValues: matrixStruct);
328 return success();
329 }
330};
331
332static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs,
333 Value rhs, bool isMin) {
334 auto floatType = cast<FloatType>(Val: getElementTypeOrSelf(type: lhs.getType()));
335 Type i1Type = builder.getI1Type();
336 if (auto vecType = dyn_cast<VectorType>(Val: lhs.getType()))
337 i1Type = VectorType::get(shape: vecType.getShape(), elementType: i1Type);
338 Value cmp = LLVM::FCmpOp::create(
339 builder, location: loc, res: i1Type,
340 predicate: isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt, lhs, rhs);
341 Value sel = LLVM::SelectOp::create(builder, location: loc, condition: cmp, trueValue: lhs, falseValue: rhs);
342 Value isNan = LLVM::FCmpOp::create(builder, location: loc, res: i1Type,
343 predicate: LLVM::FCmpPredicate::uno, lhs, rhs);
344 Value nan = LLVM::ConstantOp::create(
345 builder, location: loc, res: lhs.getType(),
346 value: builder.getFloatAttr(type: floatType,
347 value: APFloat::getQNaN(Sem: floatType.getFloatSemantics())));
348 return LLVM::SelectOp::create(builder, location: loc, condition: isNan, trueValue: nan, falseValue: sel);
349}
350
351static Value createScalarOp(OpBuilder &builder, Location loc,
352 gpu::MMAElementwiseOp op,
353 ArrayRef<Value> operands) {
354 switch (op) {
355 case gpu::MMAElementwiseOp::ADDF:
356 return LLVM::FAddOp::create(builder, location: loc, resultType: operands[0].getType(), operands);
357 case gpu::MMAElementwiseOp::MULF:
358 return LLVM::FMulOp::create(builder, location: loc, resultType: operands[0].getType(), operands);
359 case gpu::MMAElementwiseOp::DIVF:
360 return LLVM::FDivOp::create(builder, location: loc, resultType: operands[0].getType(), operands);
361 case gpu::MMAElementwiseOp::MAXF:
362 return createMinMaxF(builder, loc, lhs: operands[0], rhs: operands[1],
363 /*isMin=*/false);
364 case gpu::MMAElementwiseOp::MINF:
365 return createMinMaxF(builder, loc, lhs: operands[0], rhs: operands[1],
366 /*isMin=*/true);
367 default:
368 llvm_unreachable("unknown op");
369 }
370}
371
372/// Convert GPU MMA elementwise ops to extract + op + insert.
373struct WmmaElementwiseOpToNVVMLowering
374 : public ConvertOpToLLVMPattern<gpu::SubgroupMmaElementwiseOp> {
375 using ConvertOpToLLVMPattern<
376 gpu::SubgroupMmaElementwiseOp>::ConvertOpToLLVMPattern;
377
378 LogicalResult
379 matchAndRewrite(gpu::SubgroupMmaElementwiseOp subgroupMmaElementwiseOp,
380 OpAdaptor adaptor,
381 ConversionPatternRewriter &rewriter) const override {
382 if (failed(Result: areAllLLVMTypes(op: subgroupMmaElementwiseOp.getOperation(),
383 operands: adaptor.getOperands(), rewriter)))
384 return failure();
385 Location loc = subgroupMmaElementwiseOp.getLoc();
386 size_t numOperands = adaptor.getOperands().size();
387 Type destType = convertMMAToLLVMType(
388 type: cast<gpu::MMAMatrixType>(Val: subgroupMmaElementwiseOp.getType()));
389
390 // If the element is not a struct, it means it's a scalar f64.
391 LLVM::LLVMStructType structDestTy =
392 dyn_cast<LLVM::LLVMStructType>(Val&: destType);
393 if (!structDestTy) {
394 SmallVector<Value> operands;
395 for (auto operand : adaptor.getOperands()) {
396 operands.push_back(Elt: operand);
397 }
398 Value element = createScalarOp(
399 builder&: rewriter, loc, op: subgroupMmaElementwiseOp.getOpType(), operands);
400 rewriter.replaceOp(op: subgroupMmaElementwiseOp, newValues: element);
401 return success();
402 }
403 Value matrixStruct = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: structDestTy);
404 for (size_t i = 0, e = structDestTy.getBody().size(); i < e; ++i) {
405 SmallVector<Value> extractedOperands;
406 for (size_t opIdx = 0; opIdx < numOperands; opIdx++) {
407 extractedOperands.push_back(Elt: LLVM::ExtractValueOp::create(
408 builder&: rewriter, location: loc, container: adaptor.getOperands()[opIdx], position: i));
409 }
410 Value element =
411 createScalarOp(builder&: rewriter, loc, op: subgroupMmaElementwiseOp.getOpType(),
412 operands: extractedOperands);
413 matrixStruct =
414 LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: matrixStruct, value: element, position: i);
415 }
416 rewriter.replaceOp(op: subgroupMmaElementwiseOp, newValues: matrixStruct);
417 return success();
418 }
419};
420
421} // namespace
422
423/// Return the LLVMStructureType corresponding to the MMAMatrixType `type`.
424Type mlir::convertMMAToLLVMType(gpu::MMAMatrixType type) {
425 NVVM::MMAFrag frag = convertOperand(operandName: type.getOperand());
426 NVVM::MMATypes eltType = getElementType(type);
427 auto nRow = type.getShape()[0];
428 auto nCol = type.getShape()[1];
429 std::pair<Type, unsigned> typeInfo =
430 NVVM::inferMMAType(type: eltType, frag, nRow, nCol, context: type.getContext());
431 // Special handling for f64 a and b fragments
432 Type f64Ty = Float64Type::get(context: type.getContext());
433 if (typeInfo.first == f64Ty && typeInfo.second == 1) {
434 return f64Ty;
435 }
436 return LLVM::LLVMStructType::getLiteral(
437 context: type.getContext(), types: SmallVector<Type, 8>(typeInfo.second, typeInfo.first));
438}
439
440void mlir::populateGpuWMMAToNVVMConversionPatterns(
441 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
442 PatternBenefit benefit) {
443 patterns.add<WmmaLoadOpToNVVMLowering, WmmaMmaOpToNVVMLowering,
444 WmmaStoreOpToNVVMLowering, WmmaConstantOpToNVVMLowering,
445 WmmaElementwiseOpToNVVMLowering>(arg: converter, args&: benefit);
446}
447