| 1 | //===- MemoryAccessOpInterfacesImpl.cpp -----------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // Implement memref dialect interfaces that enable manipulating memref indexing |
| 9 | // in passes like FoldMemRefAliasOps. |
| 10 | //===----------------------------------------------------------------------===// |
| 11 | |
| 12 | #include "mlir/Dialect/AMDGPU/Transforms/MemoryAccessOpInterfacesImpl.h" |
| 13 | |
| 14 | #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h" |
| 15 | #include "mlir/Dialect/AMDGPU/Utils/MemorySpaceUtils.h" |
| 16 | #include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h" |
| 17 | #include "mlir/IR/Dialect.h" |
| 18 | #include "mlir/IR/Operation.h" |
| 19 | #include "mlir/IR/PatternMatch.h" |
| 20 | |
| 21 | using namespace mlir; |
| 22 | using namespace mlir::amdgpu; |
| 23 | using namespace mlir::memref; |
| 24 | |
| 25 | namespace { |
| 26 | template <typename OpTy> |
| 27 | struct TransposeLoadAccess final |
| 28 | : IndexedAccessOpInterface::ExternalModel<TransposeLoadAccess<OpTy>, OpTy> { |
| 29 | TypedValue<MemRefType> getAccessedMemref(Operation *op) const { |
| 30 | return cast<TypedValue<MemRefType>>(cast<OpTy>(op).getSrc()); |
| 31 | } |
| 32 | |
| 33 | Operation::operand_range getIndices(Operation *op) const { |
| 34 | return cast<OpTy>(op).getSrcIndices(); |
| 35 | } |
| 36 | |
| 37 | SmallVector<int64_t> getAccessedShape(Operation *op) const { |
| 38 | return {cast<VectorType>(cast<OpTy>(op).getResult().getType()) |
| 39 | .getNumElements()}; |
| 40 | } |
| 41 | |
| 42 | std::optional<SmallVector<Value>> |
| 43 | updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref, |
| 44 | ValueRange newIndices) const { |
| 45 | auto accessOp = cast<OpTy>(op); |
| 46 | rewriter.modifyOpInPlace(accessOp, [&]() { |
| 47 | accessOp.getSrcMutable().assign(newMemref); |
| 48 | accessOp.getSrcIndicesMutable().assign(newIndices); |
| 49 | }); |
| 50 | return std::nullopt; |
| 51 | } |
| 52 | |
| 53 | bool hasInboundsIndices(Operation *) const { return true; } |
| 54 | }; |
| 55 | |
| 56 | template <typename OpTy> |
| 57 | struct BaseAndIndicesAccess final |
| 58 | : IndexedAccessOpInterface::ExternalModel<BaseAndIndicesAccess<OpTy>, |
| 59 | OpTy> { |
| 60 | TypedValue<MemRefType> getAccessedMemref(Operation *op) const { |
| 61 | return cast<TypedValue<MemRefType>>(cast<OpTy>(op).getBase()); |
| 62 | } |
| 63 | |
| 64 | Operation::operand_range getIndices(Operation *op) const { |
| 65 | return cast<OpTy>(op).getIndices(); |
| 66 | } |
| 67 | |
| 68 | SmallVector<int64_t> getAccessedShape(Operation *) const { return {}; } |
| 69 | |
| 70 | std::optional<SmallVector<Value>> |
| 71 | updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref, |
| 72 | ValueRange newIndices) const { |
| 73 | auto accessOp = cast<OpTy>(op); |
| 74 | rewriter.modifyOpInPlace(accessOp, [&]() { |
| 75 | accessOp.getBaseMutable().assign(newMemref); |
| 76 | accessOp.getIndicesMutable().assign(newIndices); |
| 77 | }); |
| 78 | return std::nullopt; |
| 79 | } |
| 80 | |
| 81 | bool hasInboundsIndices(Operation *) const { return true; } |
| 82 | }; |
| 83 | |
| 84 | template <typename OpTy> |
| 85 | struct DescriptorAtomicBarrierAccess final |
| 86 | : IndexedAccessOpInterface::ExternalModel< |
| 87 | DescriptorAtomicBarrierAccess<OpTy>, OpTy> { |
| 88 | TypedValue<MemRefType> getAccessedMemref(Operation *op) const { |
| 89 | Value memref = cast<OpTy>(op).getAtomicBarrierAddress(); |
| 90 | if (!memref) |
| 91 | return {}; |
| 92 | return cast<TypedValue<MemRefType>>(Val&: memref); |
| 93 | } |
| 94 | |
| 95 | Operation::operand_range getIndices(Operation *op) const { |
| 96 | return cast<OpTy>(op).getAtomicBarrierIndices(); |
| 97 | } |
| 98 | |
| 99 | SmallVector<int64_t> getAccessedShape(Operation *) const { return {}; } |
| 100 | |
| 101 | std::optional<SmallVector<Value>> |
| 102 | updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref, |
| 103 | ValueRange newIndices) const { |
| 104 | auto accessOp = cast<OpTy>(op); |
| 105 | rewriter.modifyOpInPlace(accessOp, [&]() { |
| 106 | accessOp.getAtomicBarrierAddressMutable().assign(newMemref); |
| 107 | accessOp.getAtomicBarrierIndicesMutable().assign(newIndices); |
| 108 | }); |
| 109 | return std::nullopt; |
| 110 | } |
| 111 | |
| 112 | bool hasInboundsIndices(Operation *) const { return true; } |
| 113 | }; |
| 114 | |
| 115 | struct GatherToLDSCopy final |
| 116 | : IndexedMemCopyOpInterface::ExternalModel<GatherToLDSCopy, GatherToLDSOp> { |
| 117 | TypedValue<MemRefType> getSrc(Operation *op) const { |
| 118 | return cast<TypedValue<MemRefType>>(Val: cast<GatherToLDSOp>(Val: op).getSrc()); |
| 119 | } |
| 120 | |
| 121 | Operation::operand_range getSrcIndices(Operation *op) const { |
| 122 | return cast<GatherToLDSOp>(Val: op).getSrcIndices(); |
| 123 | } |
| 124 | |
| 125 | TypedValue<MemRefType> getDst(Operation *op) const { |
| 126 | return cast<TypedValue<MemRefType>>(Val: cast<GatherToLDSOp>(Val: op).getDst()); |
| 127 | } |
| 128 | |
| 129 | Operation::operand_range getDstIndices(Operation *op) const { |
| 130 | return cast<GatherToLDSOp>(Val: op).getDstIndices(); |
| 131 | } |
| 132 | |
| 133 | void setMemrefsAndIndices(Operation *op, RewriterBase &rewriter, Value newSrc, |
| 134 | ValueRange newSrcIndices, Value newDst, |
| 135 | ValueRange newDstIndices) const { |
| 136 | auto copyOp = cast<GatherToLDSOp>(Val: op); |
| 137 | rewriter.modifyOpInPlace(root: copyOp, callable: [&]() { |
| 138 | copyOp.getSrcMutable().assign(value: newSrc); |
| 139 | copyOp.getSrcIndicesMutable().assign(values: newSrcIndices); |
| 140 | copyOp.getDstMutable().assign(value: newDst); |
| 141 | copyOp.getDstIndicesMutable().assign(values: newDstIndices); |
| 142 | }); |
| 143 | } |
| 144 | |
| 145 | bool hasInboundsSrcIndices(Operation *op) const { |
| 146 | MemRefType srcType = cast<GatherToLDSOp>(Val: op).getSrc().getType(); |
| 147 | return !isFatRawBufferMemorySpace(memorySpace: srcType.getMemorySpace()); |
| 148 | } |
| 149 | |
| 150 | bool hasInboundsDstIndices(Operation *) const { return true; } |
| 151 | }; |
| 152 | |
| 153 | struct GlobalLoadAsyncToLDSCopy final |
| 154 | : IndexedMemCopyOpInterface::ExternalModel<GlobalLoadAsyncToLDSCopy, |
| 155 | GlobalLoadAsyncToLDSOp> { |
| 156 | TypedValue<MemRefType> getSrc(Operation *op) const { |
| 157 | return cast<TypedValue<MemRefType>>( |
| 158 | Val: cast<GlobalLoadAsyncToLDSOp>(Val: op).getSrc()); |
| 159 | } |
| 160 | |
| 161 | Operation::operand_range getSrcIndices(Operation *op) const { |
| 162 | return cast<GlobalLoadAsyncToLDSOp>(Val: op).getSrcIndices(); |
| 163 | } |
| 164 | |
| 165 | TypedValue<MemRefType> getDst(Operation *op) const { |
| 166 | return cast<TypedValue<MemRefType>>( |
| 167 | Val: cast<GlobalLoadAsyncToLDSOp>(Val: op).getDst()); |
| 168 | } |
| 169 | |
| 170 | Operation::operand_range getDstIndices(Operation *op) const { |
| 171 | return cast<GlobalLoadAsyncToLDSOp>(Val: op).getDstIndices(); |
| 172 | } |
| 173 | |
| 174 | void setMemrefsAndIndices(Operation *op, RewriterBase &rewriter, Value newSrc, |
| 175 | ValueRange newSrcIndices, Value newDst, |
| 176 | ValueRange newDstIndices) const { |
| 177 | auto copyOp = cast<GlobalLoadAsyncToLDSOp>(Val: op); |
| 178 | rewriter.modifyOpInPlace(root: copyOp, callable: [&]() { |
| 179 | copyOp.getSrcMutable().assign(value: newSrc); |
| 180 | copyOp.getSrcIndicesMutable().assign(values: newSrcIndices); |
| 181 | copyOp.getDstMutable().assign(value: newDst); |
| 182 | copyOp.getDstIndicesMutable().assign(values: newDstIndices); |
| 183 | }); |
| 184 | } |
| 185 | |
| 186 | bool hasInboundsSrcIndices(Operation *) const { return true; } |
| 187 | |
| 188 | bool hasInboundsDstIndices(Operation *op) const { |
| 189 | // Masked lanes may carry out-of-bounds destination indices; lowering |
| 190 | // replaces their destination pointer with -1 before the instruction uses |
| 191 | // it. |
| 192 | return !cast<GlobalLoadAsyncToLDSOp>(Val: op).getMask(); |
| 193 | } |
| 194 | }; |
| 195 | |
| 196 | template <typename OpTy> |
| 197 | struct DmaBaseCopy final |
| 198 | : IndexedMemCopyOpInterface::ExternalModel<DmaBaseCopy<OpTy>, OpTy> { |
| 199 | TypedValue<MemRefType> getSrc(Operation *op) const { |
| 200 | return cast<TypedValue<MemRefType>>(cast<OpTy>(op).getGlobal()); |
| 201 | } |
| 202 | |
| 203 | Operation::operand_range getSrcIndices(Operation *op) const { |
| 204 | return cast<OpTy>(op).getGlobalIndices(); |
| 205 | } |
| 206 | |
| 207 | TypedValue<MemRefType> getDst(Operation *op) const { |
| 208 | return cast<TypedValue<MemRefType>>(cast<OpTy>(op).getLds()); |
| 209 | } |
| 210 | |
| 211 | Operation::operand_range getDstIndices(Operation *op) const { |
| 212 | return cast<OpTy>(op).getLdsIndices(); |
| 213 | } |
| 214 | |
| 215 | void setMemrefsAndIndices(Operation *op, RewriterBase &rewriter, Value newSrc, |
| 216 | ValueRange newSrcIndices, Value newDst, |
| 217 | ValueRange newDstIndices) const { |
| 218 | auto copyOp = cast<OpTy>(op); |
| 219 | rewriter.modifyOpInPlace(copyOp, [&]() { |
| 220 | copyOp.getGlobalMutable().assign(newSrc); |
| 221 | copyOp.getGlobalIndicesMutable().assign(newSrcIndices); |
| 222 | copyOp.getLdsMutable().assign(newDst); |
| 223 | copyOp.getLdsIndicesMutable().assign(newDstIndices); |
| 224 | }); |
| 225 | } |
| 226 | |
| 227 | bool hasInboundsSrcIndices(Operation *) const { return true; } |
| 228 | |
| 229 | bool hasInboundsDstIndices(Operation *) const { return true; } |
| 230 | }; |
| 231 | } // namespace |
| 232 | |
| 233 | void mlir::amdgpu::registerMemoryAccessOpInterfacesExternalModels( |
| 234 | DialectRegistry ®istry) { |
| 235 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, amdgpu::AMDGPUDialect *) { |
| 236 | TransposeLoadOp::attachInterface<TransposeLoadAccess<TransposeLoadOp>>( |
| 237 | context&: *ctx); |
| 238 | GlobalTransposeLoadOp::attachInterface< |
| 239 | TransposeLoadAccess<GlobalTransposeLoadOp>>(context&: *ctx); |
| 240 | MakeDmaDescriptorOp::attachInterface< |
| 241 | DescriptorAtomicBarrierAccess<MakeDmaDescriptorOp>>(context&: *ctx); |
| 242 | MakeGatherDmaDescriptorOp::attachInterface< |
| 243 | DescriptorAtomicBarrierAccess<MakeGatherDmaDescriptorOp>>(context&: *ctx); |
| 244 | DsBarrierInitOp::attachInterface<BaseAndIndicesAccess<DsBarrierInitOp>>( |
| 245 | context&: *ctx); |
| 246 | DsBarrierPollStateOp::attachInterface< |
| 247 | BaseAndIndicesAccess<DsBarrierPollStateOp>>(context&: *ctx); |
| 248 | DsAsyncBarrierArriveOp::attachInterface< |
| 249 | BaseAndIndicesAccess<DsAsyncBarrierArriveOp>>(context&: *ctx); |
| 250 | DsBarrierArriveOp::attachInterface<BaseAndIndicesAccess<DsBarrierArriveOp>>( |
| 251 | context&: *ctx); |
| 252 | GatherToLDSOp::attachInterface<GatherToLDSCopy>(context&: *ctx); |
| 253 | GlobalLoadAsyncToLDSOp::attachInterface<GlobalLoadAsyncToLDSCopy>(context&: *ctx); |
| 254 | MakeDmaBaseOp::attachInterface<DmaBaseCopy<MakeDmaBaseOp>>(context&: *ctx); |
| 255 | MakeGatherDmaBaseOp::attachInterface<DmaBaseCopy<MakeGatherDmaBaseOp>>( |
| 256 | context&: *ctx); |
| 257 | }); |
| 258 | } |
| 259 | |