1//===- VectorToNVGPU.cpp - Convert vector to NVGPU dialect ----------*- C++
2//-*-===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements lowering of vector operations to NVGPU dialect ops.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/VectorToNVGPU/VectorToNVGPU.h"
15
16#include "mlir/Analysis/SliceAnalysis.h"
17#include "mlir/Analysis/TopologicalSortUtils.h"
18#include "mlir/Conversion/VectorToGPUCommon/VectorToGPUUtils.h"
19#include "mlir/Dialect/Affine/IR/AffineOps.h"
20#include "mlir/Dialect/Arith/IR/Arith.h"
21#include "mlir/Dialect/GPU/IR/GPUDialect.h"
22#include "mlir/Dialect/MemRef/IR/MemRef.h"
23#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
24#include "mlir/Dialect/NVGPU/Utils/MMAUtils.h"
25#include "mlir/Dialect/SCF/IR/SCF.h"
26#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
27#include "mlir/Dialect/Vector/IR/VectorOps.h"
28#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
29#include "mlir/IR/Builders.h"
30#include "mlir/IR/Region.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/TypeSwitch.h"
35#include "llvm/Support/DebugLog.h"
36
37#define DEBUG_TYPE "vector-to-nvgpu"
38
39namespace mlir {
40#define GEN_PASS_DEF_CONVERTVECTORTONVGPUPASS
41#include "mlir/Conversion/Passes.h.inc"
42} // namespace mlir
43
44using namespace mlir;
45
46/// For a vector TransferOpType `xferOp`, an empty `indices` vector, and an
47/// AffineMap representing offsets to apply to indices, the function fills
48/// `indices` with the original indices plus the offsets. The offsets are
49/// applied by taking into account the permutation map of the transfer op. If
50/// the `offsetMap` has dimension placeholders, those should be provided in
51/// `dimValues`.
52template <typename TransferOpType>
53static void getXferIndices(RewriterBase &rewriter, TransferOpType xferOp,
54 AffineMap offsetMap, ArrayRef<Value> dimValues,
55 SmallVector<Value, 4> &indices) {
56 indices.append(xferOp.getIndices().begin(), xferOp.getIndices().end());
57 Location loc = xferOp.getLoc();
58 unsigned offsetsIdx = 0;
59 for (auto expr : xferOp.getPermutationMap().getResults()) {
60 if (auto dim = dyn_cast<AffineDimExpr>(expr)) {
61 Value prevIdx = indices[dim.getPosition()];
62 SmallVector<OpFoldResult, 3> dims(dimValues);
63 dims.push_back(Elt: prevIdx);
64 AffineExpr d0 = rewriter.getAffineDimExpr(position: offsetMap.getNumDims());
65 indices[dim.getPosition()] = affine::makeComposedAffineApply(
66 b&: rewriter, loc, e: d0 + offsetMap.getResult(idx: offsetsIdx++), operands: dims);
67 continue;
68 }
69 }
70}
71
72// Return true if the contract op can be convert to MMA matmul.
73static bool contractSupportsMMAMatrixType(vector::ContractionOp contract) {
74 // nvgpu.mma.sync computes C += A * B, so only the `add` kind matches.
75 if (!contract.isAddKind())
76 return false;
77
78 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
79 auto infer = [&](MapList m) {
80 return AffineMap::inferFromExprList(exprsList: m, context: contract.getContext());
81 };
82 AffineExpr m, n, k;
83 bindDims(ctx: contract.getContext(), exprs&: m, exprs&: n, exprs&: k);
84 auto iteratorTypes = contract.getIteratorTypes().getValue();
85 if (!(vector::isParallelIterator(attr: iteratorTypes[0]) &&
86 vector::isParallelIterator(attr: iteratorTypes[1]) &&
87 vector::isReductionIterator(attr: iteratorTypes[2])))
88 return false;
89
90 // The contract needs to represent a matmul to be able to convert to
91 // MMAMatrix matmul.
92 if (contract.getIndexingMapsArray() != infer({{m, k}, {n, k}, {m, n}}))
93 return false;
94
95 return true;
96}
97
98/// Returns true if the extract strided slice op is supported with `mma.sync`
99/// path.
100static bool
101extractStridedSliceSupportsMMAMatrixType(vector::ExtractStridedSliceOp op) {
102
103 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
104 nvgpu::getWarpMatrixInfo(op);
105 if (failed(Result: warpMatrixInfo))
106 return false;
107
108 FailureOr<vector::ContractionOp> contractOp = nvgpu::getUserContract(op);
109 if (failed(Result: contractOp))
110 return false;
111
112 // Handle vector.extract_strided_slice on registers containing
113 // matrixB and matrixC operands. vector.extract_strided_slice op
114 // is not supported on registers containing matrixA operands.
115 if (warpMatrixInfo->operandRole == nvgpu::MatMulOperandRole::B)
116 return (cast<VectorType>(Val: op->getResult(idx: 0).getType()) ==
117 cast<VectorType>(Val: (*contractOp).getRhs().getType()));
118 if (warpMatrixInfo->operandRole == nvgpu::MatMulOperandRole::C)
119 return (cast<VectorType>(Val: op->getResult(idx: 0).getType()) ==
120 cast<VectorType>(Val: (*contractOp).getAcc().getType()));
121
122 return false;
123}
124
125static bool supportsMMaMatrixType(Operation *op) {
126 if (isa<scf::ForOp, scf::YieldOp>(Val: op))
127 return true;
128 // MMA ops cannot honor a mask. An op masked by an enclosing vector.mask has
129 // no mask operand, so the checks below would miss it; rewriting it in place
130 // would leave non-maskable ops inside the vector.mask region.
131 auto maskableOp = dyn_cast<vector::MaskableOpInterface>(Val: op);
132 if (maskableOp && maskableOp.isMasked())
133 return false;
134 if (auto transferRead = dyn_cast<vector::TransferReadOp>(Val: op))
135 return nvgpu::canLowerToWarpMatrixOperation(op: transferRead);
136 if (auto transferWrite = dyn_cast<vector::TransferWriteOp>(Val: op))
137 return nvgpu::canLowerToWarpMatrixOperation(op: transferWrite);
138 if (auto extractStridedSlice = dyn_cast<vector::ExtractStridedSliceOp>(Val: op))
139 return extractStridedSliceSupportsMMAMatrixType(op: extractStridedSlice);
140 if (auto contract = dyn_cast<vector::ContractionOp>(Val: op))
141 return contractSupportsMMAMatrixType(contract);
142 if (auto constant = dyn_cast<arith::ConstantOp>(Val: op))
143 return constantSupportsMMAMatrixType(constantOp: constant);
144 if (auto broadcast = dyn_cast<vector::BroadcastOp>(Val: op))
145 return broadcastSupportsMMAMatrixType(broadcastOp: broadcast);
146 if (auto signedExtend = dyn_cast<arith::ExtSIOp>(Val: op))
147 return integerExtendSupportsMMAMatrixType<arith::ExtSIOp>(extOp: signedExtend);
148 if (auto unsignedExtend = dyn_cast<arith::ExtUIOp>(Val: op))
149 return integerExtendSupportsMMAMatrixType<arith::ExtUIOp>(extOp: unsignedExtend);
150 return elementwiseSupportsMMAMatrixType(op);
151}
152
153// Analyze slice of operations based on convert op to figure out if the whole
154// slice can be converted to MMA operations.
155static SetVector<Operation *> getOpToConvert(mlir::Operation *op) {
156 auto hasVectorDest = [](Operation *op) {
157 return llvm::any_of(Range: op->getResultTypes(), P: llvm::IsaPred<VectorType>);
158 };
159 BackwardSliceOptions backwardSliceOptions;
160 backwardSliceOptions.filter = hasVectorDest;
161
162 auto hasVectorSrc = [](Operation *op) {
163 return llvm::any_of(Range: op->getOperandTypes(), P: llvm::IsaPred<VectorType>);
164 };
165 ForwardSliceOptions forwardSliceOptions;
166 forwardSliceOptions.filter = hasVectorSrc;
167
168 DenseMap<Operation *, SmallVector<Operation *>> backwardSliceCache;
169 DenseMap<Operation *, SmallVector<Operation *>> forwardSliceCache;
170 DenseMap<Operation *, bool> supportsMMAMatrixTypeCache;
171
172 auto getCachedBackwardSlice =
173 [&](Operation *currentOp) -> ArrayRef<Operation *> {
174 auto [it, inserted] = backwardSliceCache.try_emplace(Key: currentOp);
175 if (!inserted)
176 return it->second;
177
178 SetVector<Operation *> backwardSlice;
179 LogicalResult result =
180 getBackwardSlice(op: currentOp, backwardSlice: &backwardSlice, options: backwardSliceOptions);
181 assert(result.succeeded() && "expected a backward slice");
182 (void)result;
183 it->second = backwardSlice.takeVector();
184 return it->second;
185 };
186
187 auto getCachedForwardSlice =
188 [&](Operation *currentOp) -> ArrayRef<Operation *> {
189 auto [it, inserted] = forwardSliceCache.try_emplace(Key: currentOp);
190 if (!inserted)
191 return it->second;
192
193 SetVector<Operation *> forwardSlice;
194 // Special case for ForOp, we don't want to include the whole region but
195 // only the value using the region arguments.
196 // TODO: We should refine this to only care about the region arguments being
197 // converted to matrix type.
198 if (auto forOp = dyn_cast<scf::ForOp>(Val: currentOp)) {
199 for (Value forOpResult : forOp.getResults())
200 getForwardSlice(root: forOpResult, forwardSlice: &forwardSlice, options: forwardSliceOptions);
201 for (BlockArgument &arg : forOp.getRegionIterArgs())
202 getForwardSlice(root: arg, forwardSlice: &forwardSlice, options: forwardSliceOptions);
203 } else {
204 getForwardSlice(op: currentOp, forwardSlice: &forwardSlice, options: forwardSliceOptions);
205 }
206 it->second = forwardSlice.takeVector();
207 return it->second;
208 };
209
210 auto cachedSupportsMMAMatrixType = [&](Operation *currentOp) {
211 auto [it, inserted] =
212 supportsMMAMatrixTypeCache.try_emplace(Key: currentOp, Args: false);
213 if (inserted)
214 it->second = supportsMMaMatrixType(op: currentOp);
215 return it->second;
216 };
217
218 SetVector<Operation *> opToConvert;
219 op->walk(callback: [&](Operation *nestedOp) {
220 if (!isa<vector::ContractionOp>(Val: nestedOp) &&
221 !elementwiseSupportsMMAMatrixType(op: nestedOp))
222 return;
223 if (backwardSliceCache.contains(Val: nestedOp))
224 return;
225
226 SetVector<Operation *> dependentOps;
227 dependentOps.insert(X: nestedOp);
228 unsigned currentIndex = 0;
229 while (currentIndex != dependentOps.size()) {
230 Operation *currentOp = dependentOps[currentIndex++];
231 dependentOps.insert_range(R: getCachedBackwardSlice(currentOp));
232 dependentOps.insert_range(R: getCachedForwardSlice(currentOp));
233 }
234 // If any instruction cannot use MMA matrix type drop the whole
235 // chain. MMA matrix are stored in an opaque type so they cannot be used
236 // by all operations.
237 if (llvm::any_of(Range&: dependentOps, P: [&](Operation *op) {
238 if (!cachedSupportsMMAMatrixType(op)) {
239 LDBG() << "cannot convert op: " << *op;
240 return true;
241 }
242 return false;
243 }))
244 return;
245
246 opToConvert.insert_range(R&: dependentOps);
247 });
248 // Sort the operations so that we can convert them in topological order.
249 return topologicalSort(toSort: opToConvert);
250}
251
252/// Returns the vector type which represents a matrix fragment.
253static VectorType
254getMmaSyncVectorOperandType(const nvgpu::FragmentElementInfo &regInfo) {
255 SmallVector<int64_t> shape{regInfo.numRegistersPerFragment,
256 regInfo.elementsPerRegister};
257 Type elType = regInfo.registerLLVMType;
258 if (auto vecType = dyn_cast<VectorType>(Val&: elType))
259 elType = vecType.getElementType();
260 return VectorType::get(shape, elementType: elType);
261}
262
263/// Convert a 2D splat ConstantOp to a SubgroupMmaConstantMatrix op.
264static LogicalResult
265convertConstantOpMmaSync(RewriterBase &rewriter, arith::ConstantOp op,
266 llvm::DenseMap<Value, Value> &valueMapping) {
267 OpBuilder::InsertionGuard g(rewriter);
268 rewriter.setInsertionPoint(op);
269
270 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
271 nvgpu::getWarpMatrixInfo(op);
272 if (failed(Result: warpMatrixInfo)) {
273 LDBG() << "no warpMatrixInfo";
274 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
275 }
276
277 FailureOr<nvgpu::FragmentElementInfo> regInfo =
278 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
279 if (failed(Result: regInfo)) {
280 LDBG() << "not mma sync reg info";
281 return rewriter.notifyMatchFailure(arg&: op, msg: "not mma sync reg info");
282 }
283
284 VectorType vectorType = getMmaSyncVectorOperandType(regInfo: *regInfo);
285 auto dense = dyn_cast<SplatElementsAttr>(Val: op.getValue());
286 if (!dense) {
287 LDBG() << "not a splat";
288 return rewriter.notifyMatchFailure(arg&: op, msg: "not a splat");
289 }
290
291 Value result = arith::ConstantOp::create(
292 builder&: rewriter, location: op.getLoc(), result: vectorType,
293 value: DenseElementsAttr::get(type: vectorType, values: dense.getSplatValue<Attribute>()));
294 valueMapping[op.getResult()] = result;
295 return success();
296}
297
298/// Check if the loaded matrix operand requires transposed.
299/// Transposed Map Example:
300/// Example 1 : (..., d0, d1) -> (d1 * 1, d0 * 2)
301/// Example 2 : (d0, d1, d2, d3) -> (d3, d2)
302/// The code below checks if the output 2D is transposed using a generalized
303/// version : (d0, d1, dn, ..., dm, ...) -> (dm, dn)
304/// Returns : true; if m > n, false o.w.
305static FailureOr<bool> isTransposed(vector::TransferReadOp op) {
306 mlir::AffineMap map = op.getPermutationMap();
307
308 if (map.getNumResults() != 2) {
309 LDBG() << "Failed because the result of `vector.transfer_read` "
310 "is not a 2d operand";
311 return failure();
312 }
313
314 // Output 2D matrix dimensions in the order of d0, d1.
315 mlir::AffineExpr dM = map.getResult(idx: 0);
316 mlir::AffineExpr dN = map.getResult(idx: 1);
317
318 // Find the position of these expressions in the input.
319 auto exprM = dyn_cast<AffineDimExpr>(Val&: dM);
320 auto exprN = dyn_cast<AffineDimExpr>(Val&: dN);
321
322 if (!exprM || !exprN) {
323 LDBG() << "Failed because expressions are not affine dim "
324 "expressions, then transpose cannot be determined.";
325 return failure();
326 }
327
328 return exprM.getPosition() > exprN.getPosition();
329}
330
331static LogicalResult
332creatLdMatrixCompatibleLoads(RewriterBase &rewriter, vector::TransferReadOp op,
333 llvm::DenseMap<Value, Value> &valueMapping) {
334 OpBuilder::InsertionGuard g(rewriter);
335 rewriter.setInsertionPoint(op);
336 Location loc = op->getLoc();
337
338 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
339 nvgpu::getWarpMatrixInfo(op);
340 if (failed(Result: warpMatrixInfo)) {
341 LDBG() << "no warpMatrixInfo";
342 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
343 }
344
345 FailureOr<nvgpu::FragmentElementInfo> regInfo =
346 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
347 if (failed(Result: regInfo)) {
348 LDBG() << "not mma sync reg info";
349 return rewriter.notifyMatchFailure(arg&: op, msg: "not mma sync reg info");
350 }
351
352 FailureOr<bool> transpose = isTransposed(op);
353 if (failed(Result: transpose)) {
354 LDBG() << "failed to determine the transpose";
355 return rewriter.notifyMatchFailure(
356 arg&: op, msg: "Op should likely not be converted to a nvgpu.ldmatrix call.");
357 }
358
359 FailureOr<nvgpu::LdMatrixParams> params =
360 nvgpu::getLdMatrixParams(type: *warpMatrixInfo, transpose: *transpose);
361
362 if (failed(Result: params)) {
363 LDBG() << "failed to convert vector.transfer_read to ldmatrix. "
364 << "Op should likely not be converted to a nvgpu.ldmatrix call.";
365 return rewriter.notifyMatchFailure(
366 arg&: op, msg: "failed to convert vector.transfer_read to ldmatrix; this op "
367 "likely should not be converted to a nvgpu.ldmatrix call.");
368 }
369
370 // Adjust the load offset.
371 auto laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
372 FailureOr<AffineMap> offsets =
373 nvgpu::getLaneIdToLdMatrixMatrixCoord(builder&: rewriter, loc, params: *params);
374 if (failed(Result: offsets)) {
375 LDBG() << "no offsets";
376 return rewriter.notifyMatchFailure(arg&: op, msg: "no offsets");
377 }
378
379 VectorType vectorType = getMmaSyncVectorOperandType(regInfo: *regInfo);
380
381 SmallVector<Value, 4> indices;
382 getXferIndices<vector::TransferReadOp>(rewriter, xferOp: op, offsetMap: *offsets, dimValues: {laneId},
383 indices);
384
385 nvgpu::LdMatrixOp newOp =
386 nvgpu::LdMatrixOp::create(builder&: rewriter, location: loc, res: vectorType, srcMemref: op.getBase(),
387 indices, transpose: *transpose, numTiles: params->numTiles);
388 valueMapping[op] = newOp->getResult(idx: 0);
389 return success();
390}
391
392static LogicalResult
393createNonLdMatrixLoads(RewriterBase &rewriter, vector::TransferReadOp op,
394 llvm::DenseMap<Value, Value> &valueMapping) {
395 OpBuilder::InsertionGuard g(rewriter);
396 rewriter.setInsertionPoint(op);
397
398 Location loc = op.getLoc();
399 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
400 nvgpu::getWarpMatrixInfo(op);
401 if (failed(Result: warpMatrixInfo))
402 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
403 FailureOr<nvgpu::FragmentElementInfo> regInfo =
404 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
405 if (failed(Result: regInfo)) {
406 return rewriter.notifyMatchFailure(
407 arg&: op, msg: "Failed to deduce register fragment type during "
408 "conversion to distributed non-ldmatrix compatible load");
409 }
410
411 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
412
413 // This is the individual element type.
414 Type loadedElType = regInfo->registerLLVMType;
415 VectorType vectorType = getMmaSyncVectorOperandType(regInfo: *regInfo);
416
417 Value fill = arith::ConstantOp::create(
418 builder&: rewriter, location: op.getLoc(), result: vectorType.getElementType(),
419 value: rewriter.getZeroAttr(type: vectorType.getElementType()));
420 Value result =
421 vector::BroadcastOp::create(builder&: rewriter, location: op.getLoc(), result: vectorType, source: fill);
422
423 bool isTransposeLoad = !op.getPermutationMap().isMinorIdentity();
424
425 // If we are not transposing, then we can use vectorized loads. Otherwise, we
426 // must load each element individually.
427 if (!isTransposeLoad) {
428 if (!isa<VectorType>(Val: loadedElType)) {
429 loadedElType = VectorType::get(shape: {1}, elementType: loadedElType);
430 }
431
432 for (int i = 0; i < vectorType.getShape()[0]; i++) {
433 FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
434 builder&: rewriter, loc: op.getLoc(), fragmentType: *warpMatrixInfo);
435 if (failed(Result: coords))
436 return rewriter.notifyMatchFailure(arg&: op, msg: "no coords");
437
438 Value logicalValueId = arith::ConstantOp::create(
439 builder&: rewriter, location: loc, result: rewriter.getIndexType(),
440 value: rewriter.getIndexAttr(value: i * regInfo->elementsPerRegister));
441 SmallVector<Value, 4> newIndices;
442 getXferIndices<vector::TransferReadOp>(
443 rewriter, xferOp: op, offsetMap: *coords, dimValues: {laneId, logicalValueId}, indices&: newIndices);
444
445 Value el = vector::LoadOp::create(builder&: rewriter, location: loc, resultTypes: loadedElType,
446 base: op.getBase(), indices: newIndices);
447 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: el, dest: result, position: i);
448 }
449 } else {
450 if (auto vecType = dyn_cast<VectorType>(Val&: loadedElType)) {
451 loadedElType = vecType.getElementType();
452 }
453 for (int i = 0; i < vectorType.getShape()[0]; i++) {
454 for (unsigned innerIdx = 0; innerIdx < vectorType.getShape()[1];
455 innerIdx++) {
456
457 Value logicalValueId = arith::ConstantOp::create(
458 builder&: rewriter, location: loc, result: rewriter.getIndexType(),
459 value: rewriter.getIndexAttr(value: i * regInfo->elementsPerRegister + innerIdx));
460 FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
461 builder&: rewriter, loc: op.getLoc(), fragmentType: *warpMatrixInfo);
462 if (failed(Result: coords))
463 return rewriter.notifyMatchFailure(arg&: op, msg: "no coords");
464
465 SmallVector<Value, 4> newIndices;
466 getXferIndices<vector::TransferReadOp>(
467 rewriter, xferOp: op, offsetMap: *coords, dimValues: {laneId, logicalValueId}, indices&: newIndices);
468 Value el = memref::LoadOp::create(builder&: rewriter, location: op.getLoc(), resultType: loadedElType,
469 memref: op.getBase(), indices: newIndices);
470 result = vector::InsertOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: el, dest: result,
471 position: ArrayRef<int64_t>{i, innerIdx});
472 }
473 }
474 }
475
476 valueMapping[op.getResult()] = result;
477 return success();
478}
479
480/// Return true if this is a shared memory memref type.
481static bool isSharedMemory(MemRefType type) {
482 auto addressSpace =
483 dyn_cast_or_null<gpu::AddressSpaceAttr>(Val: type.getMemorySpace());
484 return addressSpace &&
485 addressSpace.getValue() == gpu::GPUDialect::getWorkgroupAddressSpace();
486}
487
488/// Converts a `vector.transfer_read` operation directly to either a
489/// `vector.load` or a `nvgpu.ldmatrix` operation. This function should only be
490/// used when converting to `nvgpu.mma.sync` operations.
491static LogicalResult
492convertTransferReadToLoads(RewriterBase &rewriter, vector::TransferReadOp op,
493 llvm::DenseMap<Value, Value> &valueMapping) {
494 OpBuilder::InsertionGuard g(rewriter);
495 rewriter.setInsertionPoint(op);
496
497 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
498 nvgpu::getWarpMatrixInfo(op);
499 if (failed(Result: warpMatrixInfo))
500 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
501
502 bool isLdMatrixCompatible =
503 isSharedMemory(type: cast<MemRefType>(Val: op.getBase().getType())) &&
504 nvgpu::inferTileWidthInBits(type: *warpMatrixInfo) == 128;
505
506 VectorType vecTy = op.getVectorType();
507 int64_t bitWidth = vecTy.getElementType().getIntOrFloatBitWidth();
508
509 // When we are transposing the B operand, ldmatrix will only work if we have
510 // at least 8 rows to read and the width to read for the transpose is 128
511 // bits.
512 if (!op.getPermutationMap().isMinorIdentity() &&
513 (bitWidth != 16 || vecTy.getDimSize(idx: 1) < 8 ||
514 vecTy.getDimSize(idx: 0) * bitWidth < 128))
515 isLdMatrixCompatible = false;
516
517 if (!isLdMatrixCompatible)
518 return createNonLdMatrixLoads(rewriter, op, valueMapping);
519
520 return creatLdMatrixCompatibleLoads(rewriter, op, valueMapping);
521}
522
523static LogicalResult
524convertTransferWriteToStores(RewriterBase &rewriter, vector::TransferWriteOp op,
525 llvm::DenseMap<Value, Value> &valueMapping) {
526 OpBuilder::InsertionGuard g(rewriter);
527 rewriter.setInsertionPoint(op);
528
529 Location loc = op->getLoc();
530 auto it = valueMapping.find(Val: op.getVector());
531 if (it == valueMapping.end())
532 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
533 Value matrix = it->second;
534
535 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
536 nvgpu::getWarpMatrixInfo(op);
537 if (failed(Result: warpMatrixInfo))
538 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
539 FailureOr<nvgpu::FragmentElementInfo> regInfo =
540 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
541 if (failed(Result: regInfo))
542 return rewriter.notifyMatchFailure(arg&: op, msg: "not mma sync reg info");
543
544 VectorType vectorType = getMmaSyncVectorOperandType(regInfo: *regInfo);
545 Value laneId = gpu::LaneIdOp::create(builder&: rewriter, location: loc, /*upper_bound=*/nullptr);
546
547 for (unsigned i = 0; i < vectorType.getShape()[0]; i++) {
548 Value logicalValueId = arith::ConstantOp::create(
549 builder&: rewriter, location: loc, result: rewriter.getIndexType(),
550 value: rewriter.getIndexAttr(value: i * regInfo->elementsPerRegister));
551 FailureOr<AffineMap> coords = nvgpu::getLaneIdAndValueIdToOperandCoord(
552 builder&: rewriter, loc: op.getLoc(), fragmentType: *warpMatrixInfo);
553 if (failed(Result: coords))
554 return rewriter.notifyMatchFailure(arg&: op, msg: "no coords");
555
556 Value el =
557 vector::ExtractOp::create(builder&: rewriter, location: loc, source: matrix, position: ArrayRef<int64_t>{i});
558 SmallVector<Value, 4> newIndices;
559 getXferIndices<vector::TransferWriteOp>(
560 rewriter, xferOp: op, offsetMap: *coords, dimValues: {laneId, logicalValueId}, indices&: newIndices);
561 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: el, base: op.getBase(), indices: newIndices);
562 }
563
564 LDBG() << "erase: " << op;
565 rewriter.eraseOp(op);
566 return success();
567}
568
569static void populateFromInt64AttrArray(ArrayAttr arrayAttr,
570 SmallVectorImpl<int64_t> &results) {
571 for (auto attr : arrayAttr)
572 results.push_back(Elt: cast<IntegerAttr>(Val&: attr).getInt());
573}
574
575static LogicalResult
576convertExtractStridedSlice(RewriterBase &rewriter,
577 vector::ExtractStridedSliceOp op,
578 llvm::DenseMap<Value, Value> &valueMapping) {
579 OpBuilder::InsertionGuard g(rewriter);
580 rewriter.setInsertionPoint(op);
581
582 Location loc = op->getLoc();
583
584 FailureOr<nvgpu::WarpMatrixInfo> warpMatrixInfo =
585 nvgpu::getWarpMatrixInfo(op);
586 if (failed(Result: warpMatrixInfo))
587 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
588
589 FailureOr<nvgpu::FragmentElementInfo> mmaSyncFragmentInfo =
590 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
591 if (failed(Result: mmaSyncFragmentInfo))
592 return rewriter.notifyMatchFailure(arg&: op, msg: "no mmaSyncFragmentInfo");
593
594 // Find the vector.transer_read whose result vector is being sliced.
595 auto transferReadOp = op.getSource().getDefiningOp<vector::TransferReadOp>();
596 if (!transferReadOp)
597 return rewriter.notifyMatchFailure(arg&: op, msg: "no transfer read");
598
599 warpMatrixInfo = nvgpu::getWarpMatrixInfo(op: transferReadOp);
600 if (failed(Result: warpMatrixInfo))
601 return rewriter.notifyMatchFailure(arg&: op, msg: "no warpMatrixInfo");
602
603 FailureOr<nvgpu::FragmentElementInfo> ldFragmentInfo =
604 nvgpu::getMmaSyncRegisterType(type: *warpMatrixInfo);
605 if (failed(Result: ldFragmentInfo))
606 return rewriter.notifyMatchFailure(arg&: op, msg: "no ldFragmentInfo");
607
608 assert(
609 (mmaSyncFragmentInfo->elementsPerRegister ==
610 ldFragmentInfo->elementsPerRegister) &&
611 "Number of elements per register should be same for load and mma.sync");
612
613 // Create vector.extract_strided_slice op for thread-owned fragments.
614 std::array<int64_t, 2> strides = {1,
615 1}; // stride for extract slice is always 1.
616 std::array<int64_t, 2> sliceShape = {
617 mmaSyncFragmentInfo->numRegistersPerFragment,
618 mmaSyncFragmentInfo->elementsPerRegister};
619 auto it = valueMapping.find(Val: transferReadOp);
620 if (it == valueMapping.end())
621 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
622 auto sourceVector = it->second;
623
624 // offset and sizes at warp-level of onwership.
625 SmallVector<int64_t> offsets;
626 populateFromInt64AttrArray(arrayAttr: op.getOffsets(), results&: offsets);
627
628 SmallVector<int64_t> sizes;
629 populateFromInt64AttrArray(arrayAttr: op.getSizes(), results&: sizes);
630 ArrayRef<int64_t> warpVectorShape = op.getSourceVectorType().getShape();
631
632 // Compute offset in vector registers. Note that the mma.sync vector registers
633 // are shaped as numberOfFragments x numberOfRegistersPerfFragment. The vector
634 // registers can only be sliced along numberOfFragments, i.e., sliceOffset[0].
635 std::array<int64_t, 2> sliceOffset = {0, 0};
636
637 if (offsets[0] && offsets[1])
638 return op->emitError() << "Slicing fragments in 2D is not supported. ";
639 if (offsets[0])
640 sliceOffset[0] = (warpVectorShape[0] / offsets[0]);
641 else if (offsets[1])
642 sliceOffset[0] = (warpVectorShape[1] / offsets[1]);
643
644 Value newOp = vector::ExtractStridedSliceOp::create(
645 builder&: rewriter, location: loc, source: sourceVector, offsets: sliceOffset, sizes: sliceShape, strides);
646
647 valueMapping[op] = newOp;
648 return success();
649}
650
651static LogicalResult
652convertContractOpToMmaSync(RewriterBase &rewriter, vector::ContractionOp op,
653 llvm::DenseMap<Value, Value> &valueMapping) {
654 OpBuilder::InsertionGuard g(rewriter);
655 rewriter.setInsertionPoint(op);
656
657 auto itA = valueMapping.find(Val: op.getLhs());
658 auto itB = valueMapping.find(Val: op.getRhs());
659 auto itC = valueMapping.find(Val: op.getAcc());
660 if (itA == valueMapping.end() || itB == valueMapping.end() ||
661 itC == valueMapping.end())
662 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
663 Value opA = itA->second, opB = itB->second, opC = itC->second;
664 int64_t m = cast<VectorType>(Val: op.getLhs().getType()).getShape()[0];
665 int64_t n = cast<VectorType>(Val: op.getRhs().getType()).getShape()[0];
666 int64_t k = cast<VectorType>(Val: op.getLhs().getType()).getShape()[1];
667 Value matmul = nvgpu::MmaSyncOp::create(builder&: rewriter, location: op.getLoc(), matrixA: opA, matrixB: opB, matrixC: opC,
668 mmaShape: rewriter.getI64ArrayAttr(values: {m, n, k}));
669 valueMapping[op.getResult()] = matmul;
670 return success();
671}
672
673void mlir::populatePrepareVectorToNVMMAPatterns(RewritePatternSet &patterns) {
674 vector::populateVectorContractCanonicalizeMatmulToMMT(patterns);
675 populateVectorTransferReadTransposePatterns(patterns);
676}
677
678LogicalResult mlir::convertVectorToNVVMCompatibleMMASync(RewriterBase &rewriter,
679 Operation *rootOp) {
680 SetVector<Operation *> ops = getOpToConvert(op: rootOp);
681 llvm::DenseMap<Value, Value> valueMapping;
682 for (Operation *op : ops) {
683 if (llvm::TypeSwitch<Operation *, LogicalResult>(op)
684 .Case(caseFn: [&](vector::TransferReadOp transferReadOp) {
685 return convertTransferReadToLoads(rewriter, op: transferReadOp,
686 valueMapping);
687 })
688 .Case(caseFn: [&](vector::TransferWriteOp transferWriteOp) {
689 return convertTransferWriteToStores(rewriter, op: transferWriteOp,
690 valueMapping);
691 })
692 .Case(caseFn: [&](vector::ExtractStridedSliceOp extractStridedSliceOp) {
693 return convertExtractStridedSlice(rewriter, op: extractStridedSliceOp,
694 valueMapping);
695 })
696 .Case(caseFn: [&](vector::ContractionOp contractionOp) {
697 return convertContractOpToMmaSync(rewriter, op: contractionOp,
698 valueMapping);
699 })
700 .Case(caseFn: [&](scf::ForOp forOp) {
701 return convertForOpForMMA(rewriter, op: forOp, valueMapping);
702 })
703 .Case(caseFn: [&](scf::YieldOp yieldOp) {
704 return convertYieldOpForMMA(rewriter, op: yieldOp, valueMapping);
705 })
706 .Case(caseFn: [&](arith::ConstantOp constOp) {
707 return convertConstantOpMmaSync(rewriter, op: constOp, valueMapping);
708 })
709 .Default(defaultFn: [&](Operation *op) {
710 return op->emitError() << "unhandled vector to mma type: " << *op;
711 })
712 .failed()) {
713 return op->emitOpError()
714 << "failed to convert op during vector-to-nvgpu conversion";
715 }
716 }
717 return success();
718}
719
720namespace {
721struct ConvertVectorToNVGPUPass
722 : public impl::ConvertVectorToNVGPUPassBase<ConvertVectorToNVGPUPass> {
723
724 void runOnOperation() override {
725 RewritePatternSet patterns(&getContext());
726 populatePrepareVectorToNVMMAPatterns(patterns);
727 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
728 return signalPassFailure();
729
730 IRRewriter rewriter(&getContext());
731 if (failed(Result: convertVectorToNVVMCompatibleMMASync(rewriter, rootOp: getOperation())))
732 return signalPassFailure();
733 }
734};
735} // namespace
736