1//===------ WmmaOpsToSPIRV.cpp - WMMA LD/ST/Compute to SPIRV lowering -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains definitions of patterns to lower GPU Subgroup MMA ops to
10// SPIRV Cooperative Matrix ops.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
15#include "mlir/Dialect/GPU/IR/GPUDialect.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
19#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
20#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
21#include "mlir/IR/BuiltinAttributes.h"
22#include "mlir/IR/BuiltinTypes.h"
23#include "mlir/IR/TypeUtilities.h"
24#include "mlir/IR/ValueRange.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/StringSwitch.h"
27
28#include <cassert>
29#include <limits>
30
31namespace mlir {
32//===----------------------------------------------------------------------===//
33// Patterns and helpers.
34//===----------------------------------------------------------------------===//
35
36static spirv::CooperativeMatrixType
37convertMMAMatrixType(gpu::MMAMatrixType type) {
38 ArrayRef<int64_t> shape = type.getShape();
39 auto use =
40 llvm::StringSwitch<spirv::CooperativeMatrixUseKHR>(type.getOperand())
41 .Case(S: "AOp", Value: spirv::CooperativeMatrixUseKHR::MatrixA)
42 .Case(S: "BOp", Value: spirv::CooperativeMatrixUseKHR::MatrixB)
43 .Default(Value: spirv::CooperativeMatrixUseKHR::MatrixAcc);
44 return spirv::CooperativeMatrixType::get(
45 elementType: type.getElementType(), rows: shape[0], columns: shape[1], scope: spirv::Scope::Subgroup, use);
46}
47
48// Convert a memref of `gpu.mma_matrix` into a SPIR-V pointer to an array
49// of spirv::CooperativeMatrix.
50static std::optional<Type> convertMemrefOfMMAMatrixType(MemRefType type) {
51 auto matrixType = dyn_cast<gpu::MMAMatrixType>(Val: type.getElementType());
52 if (!matrixType)
53 return std::nullopt;
54 // SPV Cooperative matrix types, and composites containing them, may only be
55 // allocated in Function or Private storage classes. For now, support only
56 // function-local arrays.
57 auto storageClass =
58 dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace());
59 if (!storageClass ||
60 storageClass.getValue() != spirv::StorageClass::Function ||
61 !type.hasStaticShape() || !type.getLayout().isIdentity() ||
62 type.getNumElements() <= 0 ||
63 type.getNumElements() > std::numeric_limits<unsigned>::max())
64 return Type();
65 auto arrayType =
66 spirv::ArrayType::get(elementType: convertMMAMatrixType(type: matrixType),
67 elementCount: static_cast<unsigned>(type.getNumElements()));
68 return spirv::PointerType::get(pointeeType: arrayType, storageClass: spirv::StorageClass::Function);
69}
70
71/// Creates a SPIR-V op to replace the given GPU subgroup mma elementwise op
72/// when the elementwise op directly supports with cooperative matrix type.
73/// Returns false if cannot.
74///
75/// See SPV_KHR_cooperative_matrix for supported elementwise ops.
76static bool createElementwiseOp(ConversionPatternRewriter &builder,
77 gpu::SubgroupMmaElementwiseOp op, Type coopType,
78 ValueRange operands) {
79 assert((isa<spirv::CooperativeMatrixType>(coopType)));
80
81 switch (op.getOpType()) {
82 case gpu::MMAElementwiseOp::ADDF:
83 builder.replaceOpWithNewOp<spirv::FAddOp>(op, args&: coopType, args&: operands);
84 return true;
85 case gpu::MMAElementwiseOp::ADDI:
86 builder.replaceOpWithNewOp<spirv::IAddOp>(op, args&: coopType, args&: operands);
87 return true;
88 case gpu::MMAElementwiseOp::SUBF:
89 builder.replaceOpWithNewOp<spirv::FSubOp>(op, args&: coopType, args&: operands);
90 return true;
91 case gpu::MMAElementwiseOp::SUBI:
92 builder.replaceOpWithNewOp<spirv::ISubOp>(op, args&: coopType, args&: operands);
93 return true;
94 case gpu::MMAElementwiseOp::MULF:
95 builder.replaceOpWithNewOp<spirv::FMulOp>(op, args&: coopType, args&: operands);
96 return true;
97 case gpu::MMAElementwiseOp::DIVF:
98 builder.replaceOpWithNewOp<spirv::FDivOp>(op, args&: coopType, args&: operands);
99 return true;
100 case gpu::MMAElementwiseOp::DIVS:
101 builder.replaceOpWithNewOp<spirv::SDivOp>(op, args&: coopType, args&: operands);
102 return true;
103 case gpu::MMAElementwiseOp::DIVU:
104 builder.replaceOpWithNewOp<spirv::UDivOp>(op, args&: coopType, args&: operands);
105 return true;
106 case gpu::MMAElementwiseOp::NEGATEF:
107 builder.replaceOpWithNewOp<spirv::FNegateOp>(op, args&: coopType, args&: operands);
108 return true;
109 case gpu::MMAElementwiseOp::NEGATES:
110 builder.replaceOpWithNewOp<spirv::SNegateOp>(op, args&: coopType, args&: operands);
111 return true;
112 case gpu::MMAElementwiseOp::EXTF:
113 case gpu::MMAElementwiseOp::TRUNCF:
114 builder.replaceOpWithNewOp<spirv::FConvertOp>(op, args&: coopType, args&: operands);
115 return true;
116 default:
117 break;
118 }
119 return false;
120}
121
122bool allOperandsHaveSameCoopMatrixType(ValueRange operands) {
123 assert(!operands.empty());
124 if (!llvm::all_equal(
125 Range: llvm::map_range(C&: operands, F: [](Value v) { return v.getType(); })))
126 return false;
127
128 return isa<spirv::CooperativeMatrixType>(Val: operands.front().getType());
129}
130
131// gpu.subgroup_mma_compute (GPUOps.td) specifies that integer A/B types carry
132// signedness and the signless accumulator implies a signed integer. Treat
133// signless C/Result as signed when selecting the SignedComponentsKHR flags.
134static bool hasSignedGpuMmaIntegerSemantics(spirv::CooperativeMatrixType type) {
135 auto elementType = dyn_cast<IntegerType>(Val: type.getElementType());
136 return elementType && !elementType.isUnsigned();
137}
138
139static spirv::CooperativeMatrixOperandsKHR
140getSignedCoopMatrixOperands(spirv::CooperativeMatrixType aType,
141 spirv::CooperativeMatrixType bType,
142 spirv::CooperativeMatrixType cType,
143 spirv::CooperativeMatrixType resultType) {
144 using Operands = spirv::CooperativeMatrixOperandsKHR;
145
146 Operands operands = Operands::None;
147 if (hasSignedGpuMmaIntegerSemantics(type: aType))
148 operands |= Operands::ASigned;
149 if (hasSignedGpuMmaIntegerSemantics(type: bType))
150 operands |= Operands::BSigned;
151 if (hasSignedGpuMmaIntegerSemantics(type: cType))
152 operands |= Operands::CSigned;
153 if (hasSignedGpuMmaIntegerSemantics(type: resultType))
154 operands |= Operands::ResultSigned;
155 return operands;
156}
157
158namespace {
159/// Converts GPU MMA ConstantMatrixOp to constant SPIR-V KHR/NV cooperative
160/// matrix ops.
161struct WmmaConstantOpToSPIRVLowering final
162 : OpConversionPattern<gpu::SubgroupMmaConstantMatrixOp> {
163 using Base::Base;
164
165 LogicalResult
166 matchAndRewrite(gpu::SubgroupMmaConstantMatrixOp op, OpAdaptor adaptor,
167 ConversionPatternRewriter &rewriter) const override {
168 Value cst = llvm::getSingleElement(C: adaptor.getOperands());
169 auto coopType = getTypeConverter()->convertType(t: op.getType());
170 if (!coopType)
171 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
172
173 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, args&: coopType, args&: cst);
174 return success();
175 }
176};
177
178/// Converts GPU MMA ExtractOp to CompositeExtract SPIR-V KHR/NV cooperative
179/// matrix ops.
180struct WmmaExtractOpToSPIRVLowering final
181 : OpConversionPattern<gpu::SubgroupMmaExtractThreadLocalOp> {
182 using Base::Base;
183
184 LogicalResult
185 matchAndRewrite(gpu::SubgroupMmaExtractThreadLocalOp op, OpAdaptor adaptor,
186 ConversionPatternRewriter &rewriter) const override {
187 Value matrix = adaptor.getMatrix();
188 auto coopType =
189 getTypeConverter()->convertType<spirv::CooperativeMatrixType>(
190 t: matrix.getType());
191 if (!coopType)
192 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
193
194 SmallVector<int32_t> intValues;
195 for (Value val : op.getIndices()) {
196 if (auto constOp = val.getDefiningOp<arith::ConstantIndexOp>()) {
197 intValues.push_back(Elt: static_cast<int32_t>(constOp.value()));
198 } else {
199 return rewriter.notifyMatchFailure(arg&: op, msg: "indices must be constants");
200 }
201 }
202
203 Type elementType = coopType.getElementType();
204 rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
205 op, args&: elementType, args&: matrix, args: rewriter.getI32ArrayAttr(values: intValues));
206 return success();
207 }
208};
209
210/// Converts GPU MMA InsertOp to CompositeInsert SPIR-V KHR/NV cooperative
211/// matrix ops.
212struct WmmaInsertOpToSPIRVLowering final
213 : OpConversionPattern<gpu::SubgroupMmaInsertThreadLocalOp> {
214 using Base::Base;
215
216 LogicalResult
217 matchAndRewrite(gpu::SubgroupMmaInsertThreadLocalOp op, OpAdaptor adaptor,
218 ConversionPatternRewriter &rewriter) const override {
219 Value value = adaptor.getValue();
220 Value matrix = adaptor.getMatrix();
221 auto coopType = getTypeConverter()->convertType(t: matrix.getType());
222 if (!coopType)
223 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
224
225 SmallVector<int32_t> intValues;
226 for (Value val : op.getIndices()) {
227 if (auto constOp = val.getDefiningOp<arith::ConstantIndexOp>()) {
228 intValues.push_back(Elt: static_cast<int32_t>(constOp.value()));
229 } else {
230 return rewriter.notifyMatchFailure(arg&: op, msg: "indices must be constants");
231 }
232 }
233
234 rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
235 op, args&: coopType, args&: value, args&: matrix, args: rewriter.getI32ArrayAttr(values: intValues));
236 return success();
237 }
238};
239
240/// Converts elementwise ops to SPIR-V cooperative matrix elementwise ops for
241/// the default case.
242struct WmmaElementwiseOpToSPIRVDefaultLowering final
243 : OpConversionPattern<gpu::SubgroupMmaElementwiseOp> {
244 using Base::Base;
245
246 LogicalResult
247 matchAndRewrite(gpu::SubgroupMmaElementwiseOp op, OpAdaptor adaptor,
248 ConversionPatternRewriter &rewriter) const override {
249 // All operands should be of cooperative matrix types.
250 if (!allOperandsHaveSameCoopMatrixType(operands: adaptor.getOperands())) {
251 return rewriter.notifyMatchFailure(arg&: op,
252 msg: "not all operands are coop matrices");
253 }
254
255 auto coopType = getTypeConverter()->convertType(t: op.getType());
256 if (!coopType)
257 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
258
259 return success(
260 IsSuccess: createElementwiseOp(builder&: rewriter, op, coopType, operands: adaptor.getOperands()));
261 }
262};
263
264/// Converts elementwise ops to SPIR-V cooperative matrix elementwise ops for
265/// matrix times scalar case.
266struct WmmaElementwiseOpToSPIRVScalarMulLowering final
267 : OpConversionPattern<gpu::SubgroupMmaElementwiseOp> {
268 using Base::Base;
269
270 LogicalResult
271 matchAndRewrite(gpu::SubgroupMmaElementwiseOp op, OpAdaptor adaptor,
272 ConversionPatternRewriter &rewriter) const override {
273 if (adaptor.getOperands().size() != 2)
274 return failure();
275
276 // All operands should be of cooperative matrix types.
277 if (!allOperandsHaveSameCoopMatrixType(operands: adaptor.getOperands())) {
278 return rewriter.notifyMatchFailure(arg&: op,
279 msg: "not all operands are coop matrices");
280 }
281
282 if (op.getOpType() != gpu::MMAElementwiseOp::MULF)
283 return failure();
284
285 // Use the original operands to check whether one of the operands is a splat
286 // scalar value.
287 Value lhs = op.getOperands().front();
288 Value rhs = op.getOperands().back();
289 Value splat = nullptr;
290 Value matrix = nullptr;
291 if (lhs.getDefiningOp<gpu::SubgroupMmaConstantMatrixOp>()) {
292 splat = adaptor.getOperands().front();
293 matrix = adaptor.getOperands().back();
294 } else if (rhs.getDefiningOp<gpu::SubgroupMmaConstantMatrixOp>()) {
295 matrix = adaptor.getOperands().front();
296 splat = adaptor.getOperands().back();
297 }
298 if (!splat || !matrix)
299 return rewriter.notifyMatchFailure(arg&: op, msg: "no splat operand");
300
301 // Constant MMA matrix ops are converted to `spirv.CompositeConstruct` ops.
302 Value scalar;
303 auto cc = splat.getDefiningOp<spirv::CompositeConstructOp>();
304 if (!cc) {
305 return rewriter.notifyMatchFailure(arg&: op,
306 msg: "splat is not a composite construct");
307 }
308
309 scalar = llvm::getSingleElement(C: cc.getConstituents());
310
311 auto coopType = getTypeConverter()->convertType(t: op.getType());
312 if (!coopType)
313 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
314 rewriter.replaceOpWithNewOp<spirv::MatrixTimesScalarOp>(
315 op, args&: coopType, args: ValueRange{matrix, scalar});
316 return success();
317 }
318};
319} // namespace
320
321//===----------------------------------------------------------------------===//
322// SPV_KHR_cooperative_matrix
323//===----------------------------------------------------------------------===//
324
325namespace khr {
326namespace {
327struct MemoryRequirements {
328 spirv::MemoryAccessAttr memoryAccess;
329 IntegerAttr alignment;
330};
331
332/// Mirrors calculateMemoryRequirements in MemRefToSPIRV.cpp, simplified since
333/// coop matrix load/store have no alignment or nontemporal attrs of their own.
334static MemoryRequirements calculateMemoryRequirements(Value accessedPtr) {
335 auto ptrType = cast<spirv::PointerType>(Val: accessedPtr.getType());
336 if (ptrType.getStorageClass() != spirv::StorageClass::PhysicalStorageBuffer)
337 return {};
338
339 // PhysicalStorageBuffer pointers require Aligned.
340 std::optional<int64_t> sizeInBytes;
341 Type pointeeType = ptrType.getPointeeType();
342 if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: pointeeType)) {
343 sizeInBytes = scalarType.getSizeInBytes();
344 } else if (auto vecType = dyn_cast<VectorType>(Val&: pointeeType)) {
345 if (auto scalarElem = dyn_cast<spirv::ScalarType>(Val: vecType.getElementType()))
346 if (auto elemSize = scalarElem.getSizeInBytes())
347 sizeInBytes = *elemSize * vecType.getNumElements();
348 }
349
350 if (!sizeInBytes)
351 return {};
352
353 MLIRContext *ctx = accessedPtr.getContext();
354 return {.memoryAccess: spirv::MemoryAccessAttr::get(context: ctx, value: spirv::MemoryAccess::Aligned),
355 .alignment: IntegerAttr::get(type: IntegerType::get(context: ctx, width: 32), value: *sizeInBytes)};
356}
357
358/// Converts the GPU MMA loadOp to KHRCooperativeMatrixLoad op in the SPIRV
359/// dialect.
360struct WmmaLoadOpToSPIRVLowering final
361 : OpConversionPattern<gpu::SubgroupMmaLoadMatrixOp> {
362 using Base::Base;
363
364 LogicalResult
365 matchAndRewrite(gpu::SubgroupMmaLoadMatrixOp op, OpAdaptor adaptor,
366 ConversionPatternRewriter &rewriter) const override {
367 const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
368 Location loc = op->getLoc();
369
370 auto retType = cast<gpu::MMAMatrixType>(Val: op.getRes().getType());
371 MemRefType memrefType = op.getSrcMemref().getType();
372 Value bufferPtr =
373 spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getSrcMemref(),
374 indices: adaptor.getIndices(), loc, builder&: rewriter);
375
376 auto coopType =
377 typeConverter.convertType<spirv::CooperativeMatrixType>(t: retType);
378 if (!coopType)
379 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
380
381 int64_t stride = op.getLeadDimension().getSExtValue();
382 IntegerType i32Type = rewriter.getI32Type();
383 auto strideValue = spirv::ConstantOp::create(
384 builder&: rewriter, location: loc, constant: i32Type, value: IntegerAttr::get(type: i32Type, value: stride));
385
386 bool isColMajor = op.getTranspose().value_or(u: false);
387 auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
388 : spirv::CooperativeMatrixLayoutKHR::RowMajor;
389
390 MemoryRequirements memReq = calculateMemoryRequirements(accessedPtr: bufferPtr);
391 rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixLoadOp>(
392 op, args&: coopType, args&: bufferPtr, args&: layout, args&: strideValue, args&: memReq.memoryAccess,
393 args&: memReq.alignment);
394 return success();
395 }
396};
397
398/// Converts the GPU MMA StoreOp to KHRCooperativeMatrixStore op in the SPIRV
399/// dialect.
400struct WmmaStoreOpToSPIRVLowering final
401 : OpConversionPattern<gpu::SubgroupMmaStoreMatrixOp> {
402 using Base::Base;
403
404 LogicalResult
405 matchAndRewrite(gpu::SubgroupMmaStoreMatrixOp op, OpAdaptor adaptor,
406 ConversionPatternRewriter &rewriter) const override {
407 const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
408 Location loc = op->getLoc();
409
410 auto memrefType = cast<MemRefType>(Val: op.getDstMemref().getType());
411 Value bufferPtr =
412 spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getDstMemref(),
413 indices: adaptor.getIndices(), loc, builder&: rewriter);
414
415 int64_t stride = op.getLeadDimension().getSExtValue();
416 IntegerType i32Type = rewriter.getI32Type();
417 auto strideValue = spirv::ConstantOp::create(
418 builder&: rewriter, location: loc, constant: i32Type, value: IntegerAttr::get(type: i32Type, value: stride));
419
420 bool isColMajor = op.getTranspose().value_or(u: false);
421 auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor
422 : spirv::CooperativeMatrixLayoutKHR::RowMajor;
423
424 MemoryRequirements memReq = calculateMemoryRequirements(accessedPtr: bufferPtr);
425 rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixStoreOp>(
426 op, args&: bufferPtr, args: adaptor.getSrc(), args&: layout, args&: strideValue,
427 args&: memReq.memoryAccess, args&: memReq.alignment);
428 return success();
429 }
430};
431
432/// Converts GPU MMA Compute to KHRCooperativeMatrixMulAdd op in the SPIRV
433/// dialect.
434struct WmmaMmaOpToSPIRVLowering final
435 : OpConversionPattern<gpu::SubgroupMmaComputeOp> {
436 using Base::Base;
437
438 LogicalResult
439 matchAndRewrite(gpu::SubgroupMmaComputeOp subgroupMmaComputeOp,
440 OpAdaptor adaptor,
441 ConversionPatternRewriter &rewriter) const override {
442 auto aType =
443 dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpA().getType());
444 auto bType =
445 dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpB().getType());
446 auto cType =
447 dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpC().getType());
448 auto resultType =
449 getTypeConverter()->convertType<spirv::CooperativeMatrixType>(
450 t: subgroupMmaComputeOp.getResult().getType());
451 if (!aType || !bType || !cType || !resultType)
452 return rewriter.notifyMatchFailure(arg&: subgroupMmaComputeOp,
453 msg: "type conversion failed");
454
455 using Operands = spirv::CooperativeMatrixOperandsKHR;
456 Operands operands =
457 getSignedCoopMatrixOperands(aType, bType, cType, resultType);
458 spirv::CooperativeMatrixOperandsKHRAttr operandsAttr;
459 if (operands != Operands::None)
460 operandsAttr = spirv::CooperativeMatrixOperandsKHRAttr::get(
461 context: rewriter.getContext(), value: operands);
462
463 rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixMulAddOp>(
464 op: subgroupMmaComputeOp, args: adaptor.getOpA(), args: adaptor.getOpB(),
465 args: adaptor.getOpC(), args&: operandsAttr);
466 return success();
467 }
468};
469
470} // namespace
471} // namespace khr
472} // namespace mlir
473
474void mlir::populateGpuWMMAToSPIRVCoopMatrixKHRConversionPatterns(
475 const SPIRVTypeConverter &converter, RewritePatternSet &patterns) {
476 using namespace mlir;
477 MLIRContext *context = patterns.getContext();
478 patterns.add<khr::WmmaLoadOpToSPIRVLowering, khr::WmmaMmaOpToSPIRVLowering,
479 khr::WmmaStoreOpToSPIRVLowering, WmmaConstantOpToSPIRVLowering,
480 WmmaExtractOpToSPIRVLowering, WmmaInsertOpToSPIRVLowering,
481 WmmaElementwiseOpToSPIRVDefaultLowering>(arg: converter, args&: context);
482 // Give the following patterns higher benefit to prevail over the default one.
483 patterns.add<WmmaElementwiseOpToSPIRVScalarMulLowering>(arg: converter, args&: context,
484 /*benefit=*/args: 2);
485}
486
487void mlir::populateMMAToSPIRVCoopMatrixTypeConversion(
488 mlir::SPIRVTypeConverter &typeConverter) {
489 typeConverter.addConversion(callback&: convertMMAMatrixType);
490 typeConverter.addConversion(callback&: convertMemrefOfMMAMatrixType);
491}
492