| 1 | //===------ WmmaOpsToNVVM.cpp - WMMA LD/ST/Compute to NVVM lowering -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains definitions of patterns to lower GPU Subgroup MMA ops to |
| 10 | // NVVM Dialect. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h" |
| 15 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 16 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMVectorOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 22 | #include "mlir/IR/TypeUtilities.h" |
| 23 | #include "mlir/IR/Types.h" |
| 24 | |
| 25 | using namespace mlir; |
| 26 | |
| 27 | namespace { |
| 28 | |
| 29 | /// Checks if all the operands of the op being lowered are of LLVM Types. The |
| 30 | /// types are expected to be converted by the `LLVMTypeConverter` before the op |
| 31 | /// is actually lowered. If the type of an operands is not already converted it |
| 32 | /// hints a missing typeConversion and failure is returned in that case. |
| 33 | static LogicalResult areAllLLVMTypes(Operation *op, ValueRange operands, |
| 34 | ConversionPatternRewriter &rewriter) { |
| 35 | if (!llvm::all_of(Range&: operands, P: [](Value value) { |
| 36 | return LLVM::isCompatibleType(type: value.getType()); |
| 37 | })) { |
| 38 | return rewriter.notifyMatchFailure( |
| 39 | arg&: op, msg: "cannot convert if operands aren't of LLVM type." ); |
| 40 | } |
| 41 | |
| 42 | return success(); |
| 43 | } |
| 44 | |
| 45 | /// Error string to emit when an unimplemented WMMA variant is encountered. |
| 46 | static constexpr StringRef kInvalidCaseStr = "Unsupported WMMA variant." ; |
| 47 | |
| 48 | static NVVM::MMAFrag convertOperand(StringRef operandName) { |
| 49 | if (operandName == "AOp" ) |
| 50 | return NVVM::MMAFrag::a; |
| 51 | if (operandName == "BOp" ) |
| 52 | return NVVM::MMAFrag::b; |
| 53 | if (operandName == "COp" ) |
| 54 | return NVVM::MMAFrag::c; |
| 55 | llvm_unreachable("Unknown operand name" ); |
| 56 | } |
| 57 | |
| 58 | static NVVM::MMATypes getElementType(gpu::MMAMatrixType type) { |
| 59 | if (type.getElementType().isF16()) |
| 60 | return NVVM::MMATypes::f16; |
| 61 | if (type.getElementType().isF32()) |
| 62 | return type.getOperand() == "COp" ? NVVM::MMATypes::f32 |
| 63 | : NVVM::MMATypes::tf32; |
| 64 | if (type.getElementType().isF64()) |
| 65 | return NVVM::MMATypes::f64; |
| 66 | if (type.getElementType().isSignedInteger(width: 8)) |
| 67 | return NVVM::MMATypes::s8; |
| 68 | if (type.getElementType().isUnsignedInteger(width: 8)) |
| 69 | return NVVM::MMATypes::u8; |
| 70 | // Accumulator type is signless and implies signed. |
| 71 | if (type.getElementType().isInteger(width: 32)) |
| 72 | return NVVM::MMATypes::s32; |
| 73 | llvm_unreachable("Unsupported type" ); |
| 74 | } |
| 75 | |
| 76 | /// This class implements the conversion of GPU MMA loadOp to wmma.load op |
| 77 | /// in the NVVM dialect. The conversion not only emits the NVVM op but also |
| 78 | /// emits code that is necessary to store the data in the destination memref |
| 79 | /// after it has been loaded. |
| 80 | struct WmmaLoadOpToNVVMLowering |
| 81 | : public ConvertOpToLLVMPattern<gpu::SubgroupMmaLoadMatrixOp> { |
| 82 | using ConvertOpToLLVMPattern< |
| 83 | gpu::SubgroupMmaLoadMatrixOp>::ConvertOpToLLVMPattern; |
| 84 | |
| 85 | LogicalResult |
| 86 | matchAndRewrite(gpu::SubgroupMmaLoadMatrixOp subgroupMmaLoadMatrixOp, |
| 87 | OpAdaptor adaptor, |
| 88 | ConversionPatternRewriter &rewriter) const override { |
| 89 | Operation *op = subgroupMmaLoadMatrixOp.getOperation(); |
| 90 | if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter))) |
| 91 | return failure(); |
| 92 | |
| 93 | // Get the shape of the MMAMatrix type being returned. The shape will |
| 94 | // choose which intrinsic this op will be lowered to. |
| 95 | NVVM::MMALayout layout = subgroupMmaLoadMatrixOp.getTranspose() |
| 96 | ? NVVM::MMALayout::col |
| 97 | : NVVM::MMALayout::row; |
| 98 | gpu::MMAMatrixType retType = |
| 99 | cast<gpu::MMAMatrixType>(Val: subgroupMmaLoadMatrixOp.getRes().getType()); |
| 100 | ArrayRef<int64_t> retTypeShape = retType.getShape(); |
| 101 | int64_t m = 0; |
| 102 | int64_t n = 0; |
| 103 | int64_t k = 0; |
| 104 | NVVM::MMATypes eltype = getElementType(type: retType); |
| 105 | // NVVM intrinsics require to give mxnxk dimensions, infer the missing |
| 106 | // dimension based on the valid intrinsics available. |
| 107 | if (retType.getOperand() == "AOp" ) { |
| 108 | m = retTypeShape[0]; |
| 109 | k = retTypeShape[1]; |
| 110 | n = NVVM::WMMALoadOp::inferNDimension(m, k, eltypeEnum: eltype); |
| 111 | } else if (retType.getOperand() == "BOp" ) { |
| 112 | k = retTypeShape[0]; |
| 113 | n = retTypeShape[1]; |
| 114 | m = NVVM::WMMALoadOp::inferMDimension(k, n, eltypeEnum: eltype); |
| 115 | } else if (retType.getOperand() == "COp" ) { |
| 116 | m = retTypeShape[0]; |
| 117 | n = retTypeShape[1]; |
| 118 | k = NVVM::WMMALoadOp::inferKDimension(m, n, eltypeEnum: eltype); |
| 119 | } |
| 120 | NVVM::MMAFrag frag = convertOperand(operandName: retType.getOperand()); |
| 121 | // Check that there is an exisiting instruction for the combination we need. |
| 122 | if (NVVM::WMMALoadOp::getIntrinsicID(m, n, k, layoutEnum: layout, eltypeEnum: eltype, fragEnum: frag) == 0) |
| 123 | return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr); |
| 124 | |
| 125 | Type resType = convertMMAToLLVMType(type: retType); |
| 126 | Location loc = op->getLoc(); |
| 127 | |
| 128 | // Create nvvm.mma_load op according to the operand types. |
| 129 | Value dataPtr = getStridedElementPtr( |
| 130 | rewriter, loc, |
| 131 | type: cast<MemRefType>(Val: subgroupMmaLoadMatrixOp.getSrcMemref().getType()), |
| 132 | memRefDesc: adaptor.getSrcMemref(), indices: adaptor.getIndices()); |
| 133 | |
| 134 | // The NVVM op takes the leading dimension as an `i32`. Reject a value that |
| 135 | // does not fit rather than silently truncating it. |
| 136 | int64_t leadDimension = |
| 137 | subgroupMmaLoadMatrixOp.getLeadDimension().getSExtValue(); |
| 138 | if (!llvm::isInt<32>(x: leadDimension)) |
| 139 | return rewriter.notifyMatchFailure( |
| 140 | arg&: op, msg: "leading dimension does not fit into an i32" ); |
| 141 | Value leadingDim = LLVM::ConstantOp::create( |
| 142 | builder&: rewriter, location: loc, res: rewriter.getI32Type(), |
| 143 | value: rewriter.getI32IntegerAttr(value: static_cast<int32_t>(leadDimension))); |
| 144 | rewriter.replaceOpWithNewOp<NVVM::WMMALoadOp>( |
| 145 | op, args&: resType, args&: dataPtr, args&: leadingDim, args&: m, args&: n, args&: k, args&: layout, args&: eltype, args&: frag); |
| 146 | return success(); |
| 147 | } |
| 148 | }; |
| 149 | |
| 150 | /// This class implements the conversion of GPU MMA storeOp to wmma.store op |
| 151 | /// in the NVVM dialect. The conversion not only emits the NVVM op but also |
| 152 | /// emits code that is necessary to unpack the data in the source and |
| 153 | /// convert the data in the format that is needed by the NVVM op. |
| 154 | struct WmmaStoreOpToNVVMLowering |
| 155 | : public ConvertOpToLLVMPattern<gpu::SubgroupMmaStoreMatrixOp> { |
| 156 | using ConvertOpToLLVMPattern< |
| 157 | gpu::SubgroupMmaStoreMatrixOp>::ConvertOpToLLVMPattern; |
| 158 | |
| 159 | LogicalResult |
| 160 | matchAndRewrite(gpu::SubgroupMmaStoreMatrixOp subgroupMmaStoreMatrixOp, |
| 161 | OpAdaptor adaptor, |
| 162 | ConversionPatternRewriter &rewriter) const override { |
| 163 | Operation *op = subgroupMmaStoreMatrixOp.getOperation(); |
| 164 | if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter))) |
| 165 | return failure(); |
| 166 | |
| 167 | Location loc = op->getLoc(); |
| 168 | |
| 169 | SmallVector<Value, 4> storeOpOperands; |
| 170 | // Get the shape of the MMAMatrix type being stored. The shape will |
| 171 | // choose which intrinsic this op will be lowered to. |
| 172 | gpu::MMAMatrixType srcType = |
| 173 | cast<gpu::MMAMatrixType>(Val: subgroupMmaStoreMatrixOp.getSrc().getType()); |
| 174 | ArrayRef<int64_t> srcTypeShape = srcType.getShape(); |
| 175 | NVVM::MMALayout layout = subgroupMmaStoreMatrixOp.getTranspose() |
| 176 | ? NVVM::MMALayout::col |
| 177 | : NVVM::MMALayout::row; |
| 178 | NVVM::MMATypes eltype = getElementType(type: srcType); |
| 179 | int64_t m = srcTypeShape[0]; |
| 180 | int64_t n = srcTypeShape[1]; |
| 181 | int64_t k = NVVM::WMMAStoreOp::inferKDimension(m, n, eltypeEnum: eltype); |
| 182 | if (NVVM::WMMAStoreOp::getIntrinsicID(m, n, k, layoutEnum: layout, eltypeEnum: eltype) == 0) |
| 183 | return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr); |
| 184 | |
| 185 | auto matrixType = cast<LLVM::LLVMStructType>(Val: adaptor.getSrc().getType()); |
| 186 | for (unsigned i = 0, e = matrixType.getBody().size(); i < e; ++i) { |
| 187 | Value toUse = |
| 188 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: adaptor.getSrc(), position: i); |
| 189 | storeOpOperands.push_back(Elt: toUse); |
| 190 | } |
| 191 | |
| 192 | Value dataPtr = getStridedElementPtr( |
| 193 | rewriter, loc, |
| 194 | type: cast<MemRefType>(Val: subgroupMmaStoreMatrixOp.getDstMemref().getType()), |
| 195 | memRefDesc: adaptor.getDstMemref(), indices: adaptor.getIndices()); |
| 196 | // The NVVM op takes the leading dimension as an `i32`. Reject a value that |
| 197 | // does not fit rather than silently truncating it. |
| 198 | int64_t leadDimension = |
| 199 | subgroupMmaStoreMatrixOp.getLeadDimension().getSExtValue(); |
| 200 | if (!llvm::isInt<32>(x: leadDimension)) |
| 201 | return rewriter.notifyMatchFailure( |
| 202 | arg&: op, msg: "leading dimension does not fit into an i32" ); |
| 203 | Value leadingDim = LLVM::ConstantOp::create( |
| 204 | builder&: rewriter, location: loc, res: rewriter.getI32Type(), |
| 205 | value: rewriter.getI32IntegerAttr(value: static_cast<int32_t>(leadDimension))); |
| 206 | rewriter.replaceOpWithNewOp<NVVM::WMMAStoreOp>( |
| 207 | op, args&: dataPtr, args&: m, args&: n, args&: k, args&: layout, args&: eltype, args&: storeOpOperands, args&: leadingDim); |
| 208 | return success(); |
| 209 | } |
| 210 | }; |
| 211 | |
| 212 | /// This class implements the conversion of GPU MMA computeOp to wmma.mma op |
| 213 | /// in the NVVM dialect. |
| 214 | struct WmmaMmaOpToNVVMLowering |
| 215 | : public ConvertOpToLLVMPattern<gpu::SubgroupMmaComputeOp> { |
| 216 | using ConvertOpToLLVMPattern< |
| 217 | gpu::SubgroupMmaComputeOp>::ConvertOpToLLVMPattern; |
| 218 | |
| 219 | LogicalResult |
| 220 | matchAndRewrite(gpu::SubgroupMmaComputeOp subgroupMmaComputeOp, |
| 221 | OpAdaptor adaptor, |
| 222 | ConversionPatternRewriter &rewriter) const override { |
| 223 | Operation *op = subgroupMmaComputeOp.getOperation(); |
| 224 | if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter))) |
| 225 | return failure(); |
| 226 | |
| 227 | Location loc = op->getLoc(); |
| 228 | |
| 229 | // The wmma.mma intrinsic in llvm requires the operands as individual |
| 230 | // values. So individual elements from the memrefs need to be extracted and |
| 231 | // then passed on to the intrinsic call. Emit llvm ops to extract individual |
| 232 | // values form lowered memrefs. |
| 233 | SmallVector<Value> unpackedOps; |
| 234 | auto unpackOp = [&](Value operand) { |
| 235 | // f64 a and b fragments are not structs but scalars. |
| 236 | if (!isa<LLVM::LLVMStructType>(Val: operand.getType())) { |
| 237 | unpackedOps.push_back(Elt: operand); |
| 238 | return; |
| 239 | } |
| 240 | // every other type is lowered to an LLVM struct, extract the values. |
| 241 | auto structType = cast<LLVM::LLVMStructType>(Val: operand.getType()); |
| 242 | for (size_t i = 0, e = structType.getBody().size(); i < e; ++i) { |
| 243 | Value toUse = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: operand, position: i); |
| 244 | unpackedOps.push_back(Elt: toUse); |
| 245 | } |
| 246 | }; |
| 247 | |
| 248 | // Get the shapes of the MMAMatrix type being used. The shapes will |
| 249 | // choose which intrinsic this op will be lowered to. |
| 250 | gpu::MMAMatrixType aType = |
| 251 | cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpA().getType()); |
| 252 | ArrayRef<int64_t> aTypeShape = aType.getShape(); |
| 253 | gpu::MMAMatrixType cType = |
| 254 | cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpC().getType()); |
| 255 | ArrayRef<int64_t> cTypeShape = cType.getShape(); |
| 256 | int64_t m = cTypeShape[0]; |
| 257 | int64_t n = cTypeShape[1]; |
| 258 | int64_t k = aTypeShape[1]; |
| 259 | NVVM::MMALayout aLayout = subgroupMmaComputeOp.getATranspose() |
| 260 | ? NVVM::MMALayout::col |
| 261 | : NVVM::MMALayout::row; |
| 262 | NVVM::MMALayout bLayout = subgroupMmaComputeOp.getBTranspose() |
| 263 | ? NVVM::MMALayout::col |
| 264 | : NVVM::MMALayout::row; |
| 265 | NVVM::MMATypes sourceType = getElementType(type: aType); |
| 266 | NVVM::MMATypes destType = getElementType(type: cType); |
| 267 | if (NVVM::WMMAMmaOp::getIntrinsicID(m, n, k, layoutAEnum: aLayout, layoutBEnum: bLayout, eltypeAEnum: sourceType, |
| 268 | eltypeBEnum: destType) == 0) |
| 269 | return rewriter.notifyMatchFailure(arg&: op, msg: kInvalidCaseStr); |
| 270 | |
| 271 | NVVM::MMATypes bElementType = getElementType( |
| 272 | type: cast<gpu::MMAMatrixType>(Val: subgroupMmaComputeOp.getOpB().getType())); |
| 273 | if (bElementType != sourceType) |
| 274 | return rewriter.notifyMatchFailure( |
| 275 | arg&: op, msg: "WMMA compute op input matrix element types must match." ); |
| 276 | |
| 277 | unpackOp(adaptor.getOpA()); |
| 278 | unpackOp(adaptor.getOpB()); |
| 279 | unpackOp(adaptor.getOpC()); |
| 280 | |
| 281 | rewriter.replaceOpWithNewOp<NVVM::WMMAMmaOp>( |
| 282 | op, args: adaptor.getOpC().getType(), args&: m, args&: n, args&: k, args&: aLayout, args&: bLayout, args&: sourceType, |
| 283 | args&: destType, args&: unpackedOps); |
| 284 | return success(); |
| 285 | } |
| 286 | }; |
| 287 | |
| 288 | /// Convert GPU MMA ConstantMatrixOp to a chain of InsertValueOp. |
| 289 | struct WmmaConstantOpToNVVMLowering |
| 290 | : public ConvertOpToLLVMPattern<gpu::SubgroupMmaConstantMatrixOp> { |
| 291 | using ConvertOpToLLVMPattern< |
| 292 | gpu::SubgroupMmaConstantMatrixOp>::ConvertOpToLLVMPattern; |
| 293 | |
| 294 | LogicalResult |
| 295 | matchAndRewrite(gpu::SubgroupMmaConstantMatrixOp subgroupMmaConstantOp, |
| 296 | OpAdaptor adaptor, |
| 297 | ConversionPatternRewriter &rewriter) const override { |
| 298 | if (failed(Result: areAllLLVMTypes(op: subgroupMmaConstantOp.getOperation(), |
| 299 | operands: adaptor.getOperands(), rewriter))) |
| 300 | return failure(); |
| 301 | Location loc = subgroupMmaConstantOp.getLoc(); |
| 302 | Value cst = adaptor.getOperands()[0]; |
| 303 | Type type = convertMMAToLLVMType( |
| 304 | type: cast<gpu::MMAMatrixType>(Val: subgroupMmaConstantOp.getType())); |
| 305 | // If the element is not a struct, it means it's a scalar f64. |
| 306 | auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: type); |
| 307 | if (!structType) { |
| 308 | rewriter.replaceOp(op: subgroupMmaConstantOp, newValues: cst); |
| 309 | return success(); |
| 310 | } |
| 311 | // If the element type is a vector create a vector from the operand. |
| 312 | if (auto vecType = dyn_cast<VectorType>(Val: structType.getBody()[0])) { |
| 313 | Value vecCst = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: vecType); |
| 314 | for (int64_t vecEl = 0; vecEl < vecType.getNumElements(); vecEl++) { |
| 315 | Value idx = LLVM::ConstantOp::create(builder&: rewriter, location: loc, |
| 316 | type: rewriter.getI32Type(), value: vecEl); |
| 317 | vecCst = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, res: vecType, vector: vecCst, |
| 318 | value: cst, position: idx); |
| 319 | } |
| 320 | cst = vecCst; |
| 321 | } |
| 322 | Value matrixStruct = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: structType); |
| 323 | for (size_t i : llvm::seq(Begin: size_t(0), End: structType.getBody().size())) { |
| 324 | matrixStruct = |
| 325 | LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: matrixStruct, value: cst, position: i); |
| 326 | } |
| 327 | rewriter.replaceOp(op: subgroupMmaConstantOp, newValues: matrixStruct); |
| 328 | return success(); |
| 329 | } |
| 330 | }; |
| 331 | |
| 332 | static Value createMinMaxF(OpBuilder &builder, Location loc, Value lhs, |
| 333 | Value rhs, bool isMin) { |
| 334 | auto floatType = cast<FloatType>(Val: getElementTypeOrSelf(type: lhs.getType())); |
| 335 | Type i1Type = builder.getI1Type(); |
| 336 | if (auto vecType = dyn_cast<VectorType>(Val: lhs.getType())) |
| 337 | i1Type = VectorType::get(shape: vecType.getShape(), elementType: i1Type); |
| 338 | Value cmp = LLVM::FCmpOp::create( |
| 339 | builder, location: loc, res: i1Type, |
| 340 | predicate: isMin ? LLVM::FCmpPredicate::olt : LLVM::FCmpPredicate::ogt, lhs, rhs); |
| 341 | Value sel = LLVM::SelectOp::create(builder, location: loc, condition: cmp, trueValue: lhs, falseValue: rhs); |
| 342 | Value isNan = LLVM::FCmpOp::create(builder, location: loc, res: i1Type, |
| 343 | predicate: LLVM::FCmpPredicate::uno, lhs, rhs); |
| 344 | Value nan = LLVM::ConstantOp::create( |
| 345 | builder, location: loc, res: lhs.getType(), |
| 346 | value: builder.getFloatAttr(type: floatType, |
| 347 | value: APFloat::getQNaN(Sem: floatType.getFloatSemantics()))); |
| 348 | return LLVM::SelectOp::create(builder, location: loc, condition: isNan, trueValue: nan, falseValue: sel); |
| 349 | } |
| 350 | |
| 351 | static Value createScalarOp(OpBuilder &builder, Location loc, |
| 352 | gpu::MMAElementwiseOp op, |
| 353 | ArrayRef<Value> operands) { |
| 354 | switch (op) { |
| 355 | case gpu::MMAElementwiseOp::ADDF: |
| 356 | return LLVM::FAddOp::create(builder, location: loc, resultType: operands[0].getType(), operands); |
| 357 | case gpu::MMAElementwiseOp::MULF: |
| 358 | return LLVM::FMulOp::create(builder, location: loc, resultType: operands[0].getType(), operands); |
| 359 | case gpu::MMAElementwiseOp::DIVF: |
| 360 | return LLVM::FDivOp::create(builder, location: loc, resultType: operands[0].getType(), operands); |
| 361 | case gpu::MMAElementwiseOp::MAXF: |
| 362 | return createMinMaxF(builder, loc, lhs: operands[0], rhs: operands[1], |
| 363 | /*isMin=*/false); |
| 364 | case gpu::MMAElementwiseOp::MINF: |
| 365 | return createMinMaxF(builder, loc, lhs: operands[0], rhs: operands[1], |
| 366 | /*isMin=*/true); |
| 367 | default: |
| 368 | llvm_unreachable("unknown op" ); |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | /// Convert GPU MMA elementwise ops to extract + op + insert. |
| 373 | struct WmmaElementwiseOpToNVVMLowering |
| 374 | : public ConvertOpToLLVMPattern<gpu::SubgroupMmaElementwiseOp> { |
| 375 | using ConvertOpToLLVMPattern< |
| 376 | gpu::SubgroupMmaElementwiseOp>::ConvertOpToLLVMPattern; |
| 377 | |
| 378 | LogicalResult |
| 379 | matchAndRewrite(gpu::SubgroupMmaElementwiseOp subgroupMmaElementwiseOp, |
| 380 | OpAdaptor adaptor, |
| 381 | ConversionPatternRewriter &rewriter) const override { |
| 382 | if (failed(Result: areAllLLVMTypes(op: subgroupMmaElementwiseOp.getOperation(), |
| 383 | operands: adaptor.getOperands(), rewriter))) |
| 384 | return failure(); |
| 385 | Location loc = subgroupMmaElementwiseOp.getLoc(); |
| 386 | size_t numOperands = adaptor.getOperands().size(); |
| 387 | Type destType = convertMMAToLLVMType( |
| 388 | type: cast<gpu::MMAMatrixType>(Val: subgroupMmaElementwiseOp.getType())); |
| 389 | |
| 390 | // If the element is not a struct, it means it's a scalar f64. |
| 391 | LLVM::LLVMStructType structDestTy = |
| 392 | dyn_cast<LLVM::LLVMStructType>(Val&: destType); |
| 393 | if (!structDestTy) { |
| 394 | SmallVector<Value> operands; |
| 395 | for (auto operand : adaptor.getOperands()) { |
| 396 | operands.push_back(Elt: operand); |
| 397 | } |
| 398 | Value element = createScalarOp( |
| 399 | builder&: rewriter, loc, op: subgroupMmaElementwiseOp.getOpType(), operands); |
| 400 | rewriter.replaceOp(op: subgroupMmaElementwiseOp, newValues: element); |
| 401 | return success(); |
| 402 | } |
| 403 | Value matrixStruct = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: structDestTy); |
| 404 | for (size_t i = 0, e = structDestTy.getBody().size(); i < e; ++i) { |
| 405 | SmallVector<Value> extractedOperands; |
| 406 | for (size_t opIdx = 0; opIdx < numOperands; opIdx++) { |
| 407 | extractedOperands.push_back(Elt: LLVM::ExtractValueOp::create( |
| 408 | builder&: rewriter, location: loc, container: adaptor.getOperands()[opIdx], position: i)); |
| 409 | } |
| 410 | Value element = |
| 411 | createScalarOp(builder&: rewriter, loc, op: subgroupMmaElementwiseOp.getOpType(), |
| 412 | operands: extractedOperands); |
| 413 | matrixStruct = |
| 414 | LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: matrixStruct, value: element, position: i); |
| 415 | } |
| 416 | rewriter.replaceOp(op: subgroupMmaElementwiseOp, newValues: matrixStruct); |
| 417 | return success(); |
| 418 | } |
| 419 | }; |
| 420 | |
| 421 | } // namespace |
| 422 | |
| 423 | /// Return the LLVMStructureType corresponding to the MMAMatrixType `type`. |
| 424 | Type mlir::convertMMAToLLVMType(gpu::MMAMatrixType type) { |
| 425 | NVVM::MMAFrag frag = convertOperand(operandName: type.getOperand()); |
| 426 | NVVM::MMATypes eltType = getElementType(type); |
| 427 | auto nRow = type.getShape()[0]; |
| 428 | auto nCol = type.getShape()[1]; |
| 429 | std::pair<Type, unsigned> typeInfo = |
| 430 | NVVM::inferMMAType(type: eltType, frag, nRow, nCol, context: type.getContext()); |
| 431 | // Special handling for f64 a and b fragments |
| 432 | Type f64Ty = Float64Type::get(context: type.getContext()); |
| 433 | if (typeInfo.first == f64Ty && typeInfo.second == 1) { |
| 434 | return f64Ty; |
| 435 | } |
| 436 | return LLVM::LLVMStructType::getLiteral( |
| 437 | context: type.getContext(), types: SmallVector<Type, 8>(typeInfo.second, typeInfo.first)); |
| 438 | } |
| 439 | |
| 440 | void mlir::populateGpuWMMAToNVVMConversionPatterns( |
| 441 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 442 | PatternBenefit benefit) { |
| 443 | patterns.add<WmmaLoadOpToNVVMLowering, WmmaMmaOpToNVVMLowering, |
| 444 | WmmaStoreOpToNVVMLowering, WmmaConstantOpToNVVMLowering, |
| 445 | WmmaElementwiseOpToNVVMLowering>(arg: converter, args&: benefit); |
| 446 | } |
| 447 | |