1//===- MemoryAccessOpInterfacesImpl.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Implement memref dialect interfaces that enable manipulating memref indexing
9// in passes like FoldMemRefAliasOps.
10//===----------------------------------------------------------------------===//
11
12#include "mlir/Dialect/NVGPU/Transforms/MemoryAccessOpInterfacesImpl.h"
13
14#include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h"
15#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
16#include "mlir/IR/Dialect.h"
17#include "mlir/IR/Operation.h"
18#include "mlir/IR/PatternMatch.h"
19
20using namespace mlir;
21using namespace mlir::memref;
22using namespace mlir::nvgpu;
23
24namespace {
25struct LdMatrixOpInterface final
26 : IndexedAccessOpInterface::ExternalModel<LdMatrixOpInterface, LdMatrixOp> {
27 TypedValue<MemRefType> getAccessedMemref(Operation *op) const {
28 return cast<LdMatrixOp>(Val: op).getSrcMemref();
29 }
30
31 Operation::operand_range getIndices(Operation *op) const {
32 return cast<LdMatrixOp>(Val: op).getIndices();
33 }
34
35 SmallVector<int64_t> getAccessedShape(Operation *op) const {
36 VectorType vecTy = cast<LdMatrixOp>(Val: op).getRes().getType();
37 // The 2-D nature of the result is an artifact of this operation returning
38 // a struct of vectors and doesn't reflect any strides that need to be
39 // preserved.
40 return SmallVector<int64_t>{vecTy.getNumElements()};
41 }
42
43 std::optional<SmallVector<Value>>
44 updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref,
45 ValueRange newIndices) const {
46 auto ldMatrixOp = cast<LdMatrixOp>(Val: op);
47 rewriter.modifyOpInPlace(root: ldMatrixOp, callable: [&]() {
48 ldMatrixOp.getSrcMemrefMutable().assign(value: newMemref);
49 ldMatrixOp.getIndicesMutable().assign(values: newIndices);
50 });
51 return std::nullopt;
52 }
53
54 bool hasInboundsIndices(Operation *) const { return true; }
55};
56
57struct DeviceAsyncCopyOpInterface final
58 : IndexedMemCopyOpInterface::ExternalModel<DeviceAsyncCopyOpInterface,
59 DeviceAsyncCopyOp> {
60 TypedValue<MemRefType> getSrc(Operation *op) const {
61 return cast<DeviceAsyncCopyOp>(Val: op).getSrc();
62 }
63
64 Operation::operand_range getSrcIndices(Operation *op) const {
65 return cast<DeviceAsyncCopyOp>(Val: op).getSrcIndices();
66 }
67
68 TypedValue<MemRefType> getDst(Operation *op) const {
69 return cast<DeviceAsyncCopyOp>(Val: op).getDst();
70 }
71
72 Operation::operand_range getDstIndices(Operation *op) const {
73 return cast<DeviceAsyncCopyOp>(Val: op).getDstIndices();
74 }
75
76 void setMemrefsAndIndices(Operation *op, RewriterBase &rewriter, Value newSrc,
77 ValueRange newSrcIndices, Value newDst,
78 ValueRange newDstIndices) const {
79 auto copyOp = cast<DeviceAsyncCopyOp>(Val: op);
80 rewriter.modifyOpInPlace(root: copyOp, callable: [&]() {
81 copyOp.getSrcMutable().assign(value: newSrc);
82 copyOp.getSrcIndicesMutable().assign(values: newSrcIndices);
83 copyOp.getDstMutable().assign(value: newDst);
84 copyOp.getDstIndicesMutable().assign(values: newDstIndices);
85 });
86 }
87};
88} // namespace
89
90void mlir::nvgpu::registerMemoryAccessOpInterfacesExternalModels(
91 DialectRegistry &registry) {
92 registry.addExtension(extensionFn: +[](MLIRContext *ctx, nvgpu::NVGPUDialect *dialect) {
93 LdMatrixOp::attachInterface<LdMatrixOpInterface>(context&: *ctx);
94 DeviceAsyncCopyOp::attachInterface<DeviceAsyncCopyOpInterface>(context&: *ctx);
95 });
96}
97