| 1 | //===------ WmmaOpsToSPIRV.cpp - WMMA LD/ST/Compute to SPIRV lowering -----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains definitions of patterns to lower GPU Subgroup MMA ops to |
| 10 | // SPIRV Cooperative Matrix ops. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h" |
| 15 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 16 | #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" |
| 17 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 18 | #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" |
| 19 | #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h" |
| 20 | #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" |
| 21 | #include "mlir/IR/BuiltinAttributes.h" |
| 22 | #include "mlir/IR/BuiltinTypes.h" |
| 23 | #include "mlir/IR/TypeUtilities.h" |
| 24 | #include "mlir/IR/ValueRange.h" |
| 25 | #include "llvm/ADT/STLExtras.h" |
| 26 | #include "llvm/ADT/StringSwitch.h" |
| 27 | |
| 28 | #include <cassert> |
| 29 | #include <limits> |
| 30 | |
| 31 | namespace mlir { |
| 32 | //===----------------------------------------------------------------------===// |
| 33 | // Patterns and helpers. |
| 34 | //===----------------------------------------------------------------------===// |
| 35 | |
| 36 | static spirv::CooperativeMatrixType |
| 37 | convertMMAMatrixType(gpu::MMAMatrixType type) { |
| 38 | ArrayRef<int64_t> shape = type.getShape(); |
| 39 | auto use = |
| 40 | llvm::StringSwitch<spirv::CooperativeMatrixUseKHR>(type.getOperand()) |
| 41 | .Case(S: "AOp" , Value: spirv::CooperativeMatrixUseKHR::MatrixA) |
| 42 | .Case(S: "BOp" , Value: spirv::CooperativeMatrixUseKHR::MatrixB) |
| 43 | .Default(Value: spirv::CooperativeMatrixUseKHR::MatrixAcc); |
| 44 | return spirv::CooperativeMatrixType::get( |
| 45 | elementType: type.getElementType(), rows: shape[0], columns: shape[1], scope: spirv::Scope::Subgroup, use); |
| 46 | } |
| 47 | |
| 48 | // Convert a memref of `gpu.mma_matrix` into a SPIR-V pointer to an array |
| 49 | // of spirv::CooperativeMatrix. |
| 50 | static std::optional<Type> convertMemrefOfMMAMatrixType(MemRefType type) { |
| 51 | auto matrixType = dyn_cast<gpu::MMAMatrixType>(Val: type.getElementType()); |
| 52 | if (!matrixType) |
| 53 | return std::nullopt; |
| 54 | // SPV Cooperative matrix types, and composites containing them, may only be |
| 55 | // allocated in Function or Private storage classes. For now, support only |
| 56 | // function-local arrays. |
| 57 | auto storageClass = |
| 58 | dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace()); |
| 59 | if (!storageClass || |
| 60 | storageClass.getValue() != spirv::StorageClass::Function || |
| 61 | !type.hasStaticShape() || !type.getLayout().isIdentity() || |
| 62 | type.getNumElements() <= 0 || |
| 63 | type.getNumElements() > std::numeric_limits<unsigned>::max()) |
| 64 | return Type(); |
| 65 | auto arrayType = |
| 66 | spirv::ArrayType::get(elementType: convertMMAMatrixType(type: matrixType), |
| 67 | elementCount: static_cast<unsigned>(type.getNumElements())); |
| 68 | return spirv::PointerType::get(pointeeType: arrayType, storageClass: spirv::StorageClass::Function); |
| 69 | } |
| 70 | |
| 71 | /// Creates a SPIR-V op to replace the given GPU subgroup mma elementwise op |
| 72 | /// when the elementwise op directly supports with cooperative matrix type. |
| 73 | /// Returns false if cannot. |
| 74 | /// |
| 75 | /// See SPV_KHR_cooperative_matrix for supported elementwise ops. |
| 76 | static bool createElementwiseOp(ConversionPatternRewriter &builder, |
| 77 | gpu::SubgroupMmaElementwiseOp op, Type coopType, |
| 78 | ValueRange operands) { |
| 79 | assert((isa<spirv::CooperativeMatrixType>(coopType))); |
| 80 | |
| 81 | switch (op.getOpType()) { |
| 82 | case gpu::MMAElementwiseOp::ADDF: |
| 83 | builder.replaceOpWithNewOp<spirv::FAddOp>(op, args&: coopType, args&: operands); |
| 84 | return true; |
| 85 | case gpu::MMAElementwiseOp::ADDI: |
| 86 | builder.replaceOpWithNewOp<spirv::IAddOp>(op, args&: coopType, args&: operands); |
| 87 | return true; |
| 88 | case gpu::MMAElementwiseOp::SUBF: |
| 89 | builder.replaceOpWithNewOp<spirv::FSubOp>(op, args&: coopType, args&: operands); |
| 90 | return true; |
| 91 | case gpu::MMAElementwiseOp::SUBI: |
| 92 | builder.replaceOpWithNewOp<spirv::ISubOp>(op, args&: coopType, args&: operands); |
| 93 | return true; |
| 94 | case gpu::MMAElementwiseOp::MULF: |
| 95 | builder.replaceOpWithNewOp<spirv::FMulOp>(op, args&: coopType, args&: operands); |
| 96 | return true; |
| 97 | case gpu::MMAElementwiseOp::DIVF: |
| 98 | builder.replaceOpWithNewOp<spirv::FDivOp>(op, args&: coopType, args&: operands); |
| 99 | return true; |
| 100 | case gpu::MMAElementwiseOp::DIVS: |
| 101 | builder.replaceOpWithNewOp<spirv::SDivOp>(op, args&: coopType, args&: operands); |
| 102 | return true; |
| 103 | case gpu::MMAElementwiseOp::DIVU: |
| 104 | builder.replaceOpWithNewOp<spirv::UDivOp>(op, args&: coopType, args&: operands); |
| 105 | return true; |
| 106 | case gpu::MMAElementwiseOp::NEGATEF: |
| 107 | builder.replaceOpWithNewOp<spirv::FNegateOp>(op, args&: coopType, args&: operands); |
| 108 | return true; |
| 109 | case gpu::MMAElementwiseOp::NEGATES: |
| 110 | builder.replaceOpWithNewOp<spirv::SNegateOp>(op, args&: coopType, args&: operands); |
| 111 | return true; |
| 112 | case gpu::MMAElementwiseOp::EXTF: |
| 113 | case gpu::MMAElementwiseOp::TRUNCF: |
| 114 | builder.replaceOpWithNewOp<spirv::FConvertOp>(op, args&: coopType, args&: operands); |
| 115 | return true; |
| 116 | default: |
| 117 | break; |
| 118 | } |
| 119 | return false; |
| 120 | } |
| 121 | |
| 122 | bool allOperandsHaveSameCoopMatrixType(ValueRange operands) { |
| 123 | assert(!operands.empty()); |
| 124 | if (!llvm::all_equal( |
| 125 | Range: llvm::map_range(C&: operands, F: [](Value v) { return v.getType(); }))) |
| 126 | return false; |
| 127 | |
| 128 | return isa<spirv::CooperativeMatrixType>(Val: operands.front().getType()); |
| 129 | } |
| 130 | |
| 131 | // gpu.subgroup_mma_compute (GPUOps.td) specifies that integer A/B types carry |
| 132 | // signedness and the signless accumulator implies a signed integer. Treat |
| 133 | // signless C/Result as signed when selecting the SignedComponentsKHR flags. |
| 134 | static bool hasSignedGpuMmaIntegerSemantics(spirv::CooperativeMatrixType type) { |
| 135 | auto elementType = dyn_cast<IntegerType>(Val: type.getElementType()); |
| 136 | return elementType && !elementType.isUnsigned(); |
| 137 | } |
| 138 | |
| 139 | static spirv::CooperativeMatrixOperandsKHR |
| 140 | getSignedCoopMatrixOperands(spirv::CooperativeMatrixType aType, |
| 141 | spirv::CooperativeMatrixType bType, |
| 142 | spirv::CooperativeMatrixType cType, |
| 143 | spirv::CooperativeMatrixType resultType) { |
| 144 | using Operands = spirv::CooperativeMatrixOperandsKHR; |
| 145 | |
| 146 | Operands operands = Operands::None; |
| 147 | if (hasSignedGpuMmaIntegerSemantics(type: aType)) |
| 148 | operands |= Operands::ASigned; |
| 149 | if (hasSignedGpuMmaIntegerSemantics(type: bType)) |
| 150 | operands |= Operands::BSigned; |
| 151 | if (hasSignedGpuMmaIntegerSemantics(type: cType)) |
| 152 | operands |= Operands::CSigned; |
| 153 | if (hasSignedGpuMmaIntegerSemantics(type: resultType)) |
| 154 | operands |= Operands::ResultSigned; |
| 155 | return operands; |
| 156 | } |
| 157 | |
| 158 | namespace { |
| 159 | /// Converts GPU MMA ConstantMatrixOp to constant SPIR-V KHR/NV cooperative |
| 160 | /// matrix ops. |
| 161 | struct WmmaConstantOpToSPIRVLowering final |
| 162 | : OpConversionPattern<gpu::SubgroupMmaConstantMatrixOp> { |
| 163 | using Base::Base; |
| 164 | |
| 165 | LogicalResult |
| 166 | matchAndRewrite(gpu::SubgroupMmaConstantMatrixOp op, OpAdaptor adaptor, |
| 167 | ConversionPatternRewriter &rewriter) const override { |
| 168 | Value cst = llvm::getSingleElement(C: adaptor.getOperands()); |
| 169 | auto coopType = getTypeConverter()->convertType(t: op.getType()); |
| 170 | if (!coopType) |
| 171 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 172 | |
| 173 | rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, args&: coopType, args&: cst); |
| 174 | return success(); |
| 175 | } |
| 176 | }; |
| 177 | |
| 178 | /// Converts GPU MMA ExtractOp to CompositeExtract SPIR-V KHR/NV cooperative |
| 179 | /// matrix ops. |
| 180 | struct final |
| 181 | : OpConversionPattern<gpu::SubgroupMmaExtractThreadLocalOp> { |
| 182 | using Base::Base; |
| 183 | |
| 184 | LogicalResult |
| 185 | matchAndRewrite(gpu::SubgroupMmaExtractThreadLocalOp op, OpAdaptor adaptor, |
| 186 | ConversionPatternRewriter &rewriter) const override { |
| 187 | Value matrix = adaptor.getMatrix(); |
| 188 | auto coopType = |
| 189 | getTypeConverter()->convertType<spirv::CooperativeMatrixType>( |
| 190 | t: matrix.getType()); |
| 191 | if (!coopType) |
| 192 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 193 | |
| 194 | SmallVector<int32_t> intValues; |
| 195 | for (Value val : op.getIndices()) { |
| 196 | if (auto constOp = val.getDefiningOp<arith::ConstantIndexOp>()) { |
| 197 | intValues.push_back(Elt: static_cast<int32_t>(constOp.value())); |
| 198 | } else { |
| 199 | return rewriter.notifyMatchFailure(arg&: op, msg: "indices must be constants" ); |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | Type elementType = coopType.getElementType(); |
| 204 | rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>( |
| 205 | op, args&: elementType, args&: matrix, args: rewriter.getI32ArrayAttr(values: intValues)); |
| 206 | return success(); |
| 207 | } |
| 208 | }; |
| 209 | |
| 210 | /// Converts GPU MMA InsertOp to CompositeInsert SPIR-V KHR/NV cooperative |
| 211 | /// matrix ops. |
| 212 | struct WmmaInsertOpToSPIRVLowering final |
| 213 | : OpConversionPattern<gpu::SubgroupMmaInsertThreadLocalOp> { |
| 214 | using Base::Base; |
| 215 | |
| 216 | LogicalResult |
| 217 | matchAndRewrite(gpu::SubgroupMmaInsertThreadLocalOp op, OpAdaptor adaptor, |
| 218 | ConversionPatternRewriter &rewriter) const override { |
| 219 | Value value = adaptor.getValue(); |
| 220 | Value matrix = adaptor.getMatrix(); |
| 221 | auto coopType = getTypeConverter()->convertType(t: matrix.getType()); |
| 222 | if (!coopType) |
| 223 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 224 | |
| 225 | SmallVector<int32_t> intValues; |
| 226 | for (Value val : op.getIndices()) { |
| 227 | if (auto constOp = val.getDefiningOp<arith::ConstantIndexOp>()) { |
| 228 | intValues.push_back(Elt: static_cast<int32_t>(constOp.value())); |
| 229 | } else { |
| 230 | return rewriter.notifyMatchFailure(arg&: op, msg: "indices must be constants" ); |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>( |
| 235 | op, args&: coopType, args&: value, args&: matrix, args: rewriter.getI32ArrayAttr(values: intValues)); |
| 236 | return success(); |
| 237 | } |
| 238 | }; |
| 239 | |
| 240 | /// Converts elementwise ops to SPIR-V cooperative matrix elementwise ops for |
| 241 | /// the default case. |
| 242 | struct WmmaElementwiseOpToSPIRVDefaultLowering final |
| 243 | : OpConversionPattern<gpu::SubgroupMmaElementwiseOp> { |
| 244 | using Base::Base; |
| 245 | |
| 246 | LogicalResult |
| 247 | matchAndRewrite(gpu::SubgroupMmaElementwiseOp op, OpAdaptor adaptor, |
| 248 | ConversionPatternRewriter &rewriter) const override { |
| 249 | // All operands should be of cooperative matrix types. |
| 250 | if (!allOperandsHaveSameCoopMatrixType(operands: adaptor.getOperands())) { |
| 251 | return rewriter.notifyMatchFailure(arg&: op, |
| 252 | msg: "not all operands are coop matrices" ); |
| 253 | } |
| 254 | |
| 255 | auto coopType = getTypeConverter()->convertType(t: op.getType()); |
| 256 | if (!coopType) |
| 257 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 258 | |
| 259 | return success( |
| 260 | IsSuccess: createElementwiseOp(builder&: rewriter, op, coopType, operands: adaptor.getOperands())); |
| 261 | } |
| 262 | }; |
| 263 | |
| 264 | /// Converts elementwise ops to SPIR-V cooperative matrix elementwise ops for |
| 265 | /// matrix times scalar case. |
| 266 | struct WmmaElementwiseOpToSPIRVScalarMulLowering final |
| 267 | : OpConversionPattern<gpu::SubgroupMmaElementwiseOp> { |
| 268 | using Base::Base; |
| 269 | |
| 270 | LogicalResult |
| 271 | matchAndRewrite(gpu::SubgroupMmaElementwiseOp op, OpAdaptor adaptor, |
| 272 | ConversionPatternRewriter &rewriter) const override { |
| 273 | if (adaptor.getOperands().size() != 2) |
| 274 | return failure(); |
| 275 | |
| 276 | // All operands should be of cooperative matrix types. |
| 277 | if (!allOperandsHaveSameCoopMatrixType(operands: adaptor.getOperands())) { |
| 278 | return rewriter.notifyMatchFailure(arg&: op, |
| 279 | msg: "not all operands are coop matrices" ); |
| 280 | } |
| 281 | |
| 282 | if (op.getOpType() != gpu::MMAElementwiseOp::MULF) |
| 283 | return failure(); |
| 284 | |
| 285 | // Use the original operands to check whether one of the operands is a splat |
| 286 | // scalar value. |
| 287 | Value lhs = op.getOperands().front(); |
| 288 | Value rhs = op.getOperands().back(); |
| 289 | Value splat = nullptr; |
| 290 | Value matrix = nullptr; |
| 291 | if (lhs.getDefiningOp<gpu::SubgroupMmaConstantMatrixOp>()) { |
| 292 | splat = adaptor.getOperands().front(); |
| 293 | matrix = adaptor.getOperands().back(); |
| 294 | } else if (rhs.getDefiningOp<gpu::SubgroupMmaConstantMatrixOp>()) { |
| 295 | matrix = adaptor.getOperands().front(); |
| 296 | splat = adaptor.getOperands().back(); |
| 297 | } |
| 298 | if (!splat || !matrix) |
| 299 | return rewriter.notifyMatchFailure(arg&: op, msg: "no splat operand" ); |
| 300 | |
| 301 | // Constant MMA matrix ops are converted to `spirv.CompositeConstruct` ops. |
| 302 | Value scalar; |
| 303 | auto cc = splat.getDefiningOp<spirv::CompositeConstructOp>(); |
| 304 | if (!cc) { |
| 305 | return rewriter.notifyMatchFailure(arg&: op, |
| 306 | msg: "splat is not a composite construct" ); |
| 307 | } |
| 308 | |
| 309 | scalar = llvm::getSingleElement(C: cc.getConstituents()); |
| 310 | |
| 311 | auto coopType = getTypeConverter()->convertType(t: op.getType()); |
| 312 | if (!coopType) |
| 313 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 314 | rewriter.replaceOpWithNewOp<spirv::MatrixTimesScalarOp>( |
| 315 | op, args&: coopType, args: ValueRange{matrix, scalar}); |
| 316 | return success(); |
| 317 | } |
| 318 | }; |
| 319 | } // namespace |
| 320 | |
| 321 | //===----------------------------------------------------------------------===// |
| 322 | // SPV_KHR_cooperative_matrix |
| 323 | //===----------------------------------------------------------------------===// |
| 324 | |
| 325 | namespace khr { |
| 326 | namespace { |
| 327 | struct MemoryRequirements { |
| 328 | spirv::MemoryAccessAttr memoryAccess; |
| 329 | IntegerAttr alignment; |
| 330 | }; |
| 331 | |
| 332 | /// Mirrors calculateMemoryRequirements in MemRefToSPIRV.cpp, simplified since |
| 333 | /// coop matrix load/store have no alignment or nontemporal attrs of their own. |
| 334 | static MemoryRequirements calculateMemoryRequirements(Value accessedPtr) { |
| 335 | auto ptrType = cast<spirv::PointerType>(Val: accessedPtr.getType()); |
| 336 | if (ptrType.getStorageClass() != spirv::StorageClass::PhysicalStorageBuffer) |
| 337 | return {}; |
| 338 | |
| 339 | // PhysicalStorageBuffer pointers require Aligned. |
| 340 | std::optional<int64_t> sizeInBytes; |
| 341 | Type pointeeType = ptrType.getPointeeType(); |
| 342 | if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: pointeeType)) { |
| 343 | sizeInBytes = scalarType.getSizeInBytes(); |
| 344 | } else if (auto vecType = dyn_cast<VectorType>(Val&: pointeeType)) { |
| 345 | if (auto scalarElem = dyn_cast<spirv::ScalarType>(Val: vecType.getElementType())) |
| 346 | if (auto elemSize = scalarElem.getSizeInBytes()) |
| 347 | sizeInBytes = *elemSize * vecType.getNumElements(); |
| 348 | } |
| 349 | |
| 350 | if (!sizeInBytes) |
| 351 | return {}; |
| 352 | |
| 353 | MLIRContext *ctx = accessedPtr.getContext(); |
| 354 | return {.memoryAccess: spirv::MemoryAccessAttr::get(context: ctx, value: spirv::MemoryAccess::Aligned), |
| 355 | .alignment: IntegerAttr::get(type: IntegerType::get(context: ctx, width: 32), value: *sizeInBytes)}; |
| 356 | } |
| 357 | |
| 358 | /// Converts the GPU MMA loadOp to KHRCooperativeMatrixLoad op in the SPIRV |
| 359 | /// dialect. |
| 360 | struct WmmaLoadOpToSPIRVLowering final |
| 361 | : OpConversionPattern<gpu::SubgroupMmaLoadMatrixOp> { |
| 362 | using Base::Base; |
| 363 | |
| 364 | LogicalResult |
| 365 | matchAndRewrite(gpu::SubgroupMmaLoadMatrixOp op, OpAdaptor adaptor, |
| 366 | ConversionPatternRewriter &rewriter) const override { |
| 367 | const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 368 | Location loc = op->getLoc(); |
| 369 | |
| 370 | auto retType = cast<gpu::MMAMatrixType>(Val: op.getRes().getType()); |
| 371 | MemRefType memrefType = op.getSrcMemref().getType(); |
| 372 | Value bufferPtr = |
| 373 | spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getSrcMemref(), |
| 374 | indices: adaptor.getIndices(), loc, builder&: rewriter); |
| 375 | |
| 376 | auto coopType = |
| 377 | typeConverter.convertType<spirv::CooperativeMatrixType>(t: retType); |
| 378 | if (!coopType) |
| 379 | return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed" ); |
| 380 | |
| 381 | int64_t stride = op.getLeadDimension().getSExtValue(); |
| 382 | IntegerType i32Type = rewriter.getI32Type(); |
| 383 | auto strideValue = spirv::ConstantOp::create( |
| 384 | builder&: rewriter, location: loc, constant: i32Type, value: IntegerAttr::get(type: i32Type, value: stride)); |
| 385 | |
| 386 | bool isColMajor = op.getTranspose().value_or(u: false); |
| 387 | auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor |
| 388 | : spirv::CooperativeMatrixLayoutKHR::RowMajor; |
| 389 | |
| 390 | MemoryRequirements memReq = calculateMemoryRequirements(accessedPtr: bufferPtr); |
| 391 | rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixLoadOp>( |
| 392 | op, args&: coopType, args&: bufferPtr, args&: layout, args&: strideValue, args&: memReq.memoryAccess, |
| 393 | args&: memReq.alignment); |
| 394 | return success(); |
| 395 | } |
| 396 | }; |
| 397 | |
| 398 | /// Converts the GPU MMA StoreOp to KHRCooperativeMatrixStore op in the SPIRV |
| 399 | /// dialect. |
| 400 | struct WmmaStoreOpToSPIRVLowering final |
| 401 | : OpConversionPattern<gpu::SubgroupMmaStoreMatrixOp> { |
| 402 | using Base::Base; |
| 403 | |
| 404 | LogicalResult |
| 405 | matchAndRewrite(gpu::SubgroupMmaStoreMatrixOp op, OpAdaptor adaptor, |
| 406 | ConversionPatternRewriter &rewriter) const override { |
| 407 | const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 408 | Location loc = op->getLoc(); |
| 409 | |
| 410 | auto memrefType = cast<MemRefType>(Val: op.getDstMemref().getType()); |
| 411 | Value bufferPtr = |
| 412 | spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getDstMemref(), |
| 413 | indices: adaptor.getIndices(), loc, builder&: rewriter); |
| 414 | |
| 415 | int64_t stride = op.getLeadDimension().getSExtValue(); |
| 416 | IntegerType i32Type = rewriter.getI32Type(); |
| 417 | auto strideValue = spirv::ConstantOp::create( |
| 418 | builder&: rewriter, location: loc, constant: i32Type, value: IntegerAttr::get(type: i32Type, value: stride)); |
| 419 | |
| 420 | bool isColMajor = op.getTranspose().value_or(u: false); |
| 421 | auto layout = isColMajor ? spirv::CooperativeMatrixLayoutKHR::ColumnMajor |
| 422 | : spirv::CooperativeMatrixLayoutKHR::RowMajor; |
| 423 | |
| 424 | MemoryRequirements memReq = calculateMemoryRequirements(accessedPtr: bufferPtr); |
| 425 | rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixStoreOp>( |
| 426 | op, args&: bufferPtr, args: adaptor.getSrc(), args&: layout, args&: strideValue, |
| 427 | args&: memReq.memoryAccess, args&: memReq.alignment); |
| 428 | return success(); |
| 429 | } |
| 430 | }; |
| 431 | |
| 432 | /// Converts GPU MMA Compute to KHRCooperativeMatrixMulAdd op in the SPIRV |
| 433 | /// dialect. |
| 434 | struct WmmaMmaOpToSPIRVLowering final |
| 435 | : OpConversionPattern<gpu::SubgroupMmaComputeOp> { |
| 436 | using Base::Base; |
| 437 | |
| 438 | LogicalResult |
| 439 | matchAndRewrite(gpu::SubgroupMmaComputeOp subgroupMmaComputeOp, |
| 440 | OpAdaptor adaptor, |
| 441 | ConversionPatternRewriter &rewriter) const override { |
| 442 | auto aType = |
| 443 | dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpA().getType()); |
| 444 | auto bType = |
| 445 | dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpB().getType()); |
| 446 | auto cType = |
| 447 | dyn_cast<spirv::CooperativeMatrixType>(Val: adaptor.getOpC().getType()); |
| 448 | auto resultType = |
| 449 | getTypeConverter()->convertType<spirv::CooperativeMatrixType>( |
| 450 | t: subgroupMmaComputeOp.getResult().getType()); |
| 451 | if (!aType || !bType || !cType || !resultType) |
| 452 | return rewriter.notifyMatchFailure(arg&: subgroupMmaComputeOp, |
| 453 | msg: "type conversion failed" ); |
| 454 | |
| 455 | using Operands = spirv::CooperativeMatrixOperandsKHR; |
| 456 | Operands operands = |
| 457 | getSignedCoopMatrixOperands(aType, bType, cType, resultType); |
| 458 | spirv::CooperativeMatrixOperandsKHRAttr operandsAttr; |
| 459 | if (operands != Operands::None) |
| 460 | operandsAttr = spirv::CooperativeMatrixOperandsKHRAttr::get( |
| 461 | context: rewriter.getContext(), value: operands); |
| 462 | |
| 463 | rewriter.replaceOpWithNewOp<spirv::KHRCooperativeMatrixMulAddOp>( |
| 464 | op: subgroupMmaComputeOp, args: adaptor.getOpA(), args: adaptor.getOpB(), |
| 465 | args: adaptor.getOpC(), args&: operandsAttr); |
| 466 | return success(); |
| 467 | } |
| 468 | }; |
| 469 | |
| 470 | } // namespace |
| 471 | } // namespace khr |
| 472 | } // namespace mlir |
| 473 | |
| 474 | void mlir::populateGpuWMMAToSPIRVCoopMatrixKHRConversionPatterns( |
| 475 | const SPIRVTypeConverter &converter, RewritePatternSet &patterns) { |
| 476 | using namespace mlir; |
| 477 | MLIRContext *context = patterns.getContext(); |
| 478 | patterns.add<khr::WmmaLoadOpToSPIRVLowering, khr::WmmaMmaOpToSPIRVLowering, |
| 479 | khr::WmmaStoreOpToSPIRVLowering, WmmaConstantOpToSPIRVLowering, |
| 480 | WmmaExtractOpToSPIRVLowering, WmmaInsertOpToSPIRVLowering, |
| 481 | WmmaElementwiseOpToSPIRVDefaultLowering>(arg: converter, args&: context); |
| 482 | // Give the following patterns higher benefit to prevail over the default one. |
| 483 | patterns.add<WmmaElementwiseOpToSPIRVScalarMulLowering>(arg: converter, args&: context, |
| 484 | /*benefit=*/args: 2); |
| 485 | } |
| 486 | |
| 487 | void mlir::populateMMAToSPIRVCoopMatrixTypeConversion( |
| 488 | mlir::SPIRVTypeConverter &typeConverter) { |
| 489 | typeConverter.addConversion(callback&: convertMMAMatrixType); |
| 490 | typeConverter.addConversion(callback&: convertMemrefOfMMAMatrixType); |
| 491 | } |
| 492 | |