| 1 | //===-ElideReinterpretCast.cpp - Expansion patterns for MemRef operations-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 11 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 12 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 13 | #include "mlir/Dialect/MemRef/Transforms/Transforms.h" |
| 14 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 15 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 16 | #include "mlir/IR/Matchers.h" |
| 17 | #include "mlir/IR/TypeUtilities.h" |
| 18 | #include "mlir/Transforms/DialectConversion.h" |
| 19 | #include <array> |
| 20 | #include <cassert> |
| 21 | #include <optional> |
| 22 | |
| 23 | namespace mlir { |
| 24 | namespace memref { |
| 25 | #define GEN_PASS_DEF_ELIDEREINTERPRETCASTPASS |
| 26 | #include "mlir/Dialect/MemRef/Transforms/Passes.h.inc" |
| 27 | } // namespace memref |
| 28 | } // namespace mlir |
| 29 | |
| 30 | using namespace mlir; |
| 31 | |
| 32 | namespace { |
| 33 | |
| 34 | //===----------------------------------------------------------------------===// |
| 35 | // Copy Rewrite Helpers |
| 36 | //===----------------------------------------------------------------------===// |
| 37 | |
| 38 | /// Copy-relevant information derived from a reinterpret_cast. |
| 39 | struct ResultNonUnitDimsAndOffsetsForRC { |
| 40 | // Non-unit dimensions of the reinterpret_cast result. |
| 41 | SmallVector<unsigned> nonUnitDimsPos; |
| 42 | // Delinearized offsets to in-bounds reinterpret_cast source indices. |
| 43 | // Optional since it is only supported for static offsets. |
| 44 | std::optional<SmallVector<int64_t>> delinearizedOffsets; |
| 45 | }; |
| 46 | |
| 47 | /// Returns delinearized offset indices for a static reinterpret_cast offset of |
| 48 | /// an identity-layout source. |
| 49 | static std::optional<SmallVector<int64_t>> |
| 50 | delinearizeStaticRCOffset(memref::ReinterpretCastOp rc) { |
| 51 | ArrayRef<int64_t> rcOffsets = rc.getStaticOffsets(); |
| 52 | MemRefType srcType = dyn_cast<MemRefType>(Val: rc.getSource().getType()); |
| 53 | // FIXME: Despite what `getStaticOffsets` implies, `reinterpret_cast` takes |
| 54 | // only a single offset. That should be fixed at the op definition level. |
| 55 | assert(rcOffsets.size() == 1 && "Expecting single offset" ); |
| 56 | |
| 57 | assert(ShapedType::isStatic(rcOffsets[0]) && "expected static offset" ); |
| 58 | assert(rcOffsets[0] >= 0 && |
| 59 | "static reinterpret_cast offset must be non-negative" ); |
| 60 | assert(srcType.getLayout().isIdentity() && |
| 61 | "Expecting identity source layout." ); |
| 62 | if (srcType.getRank() == 0) { |
| 63 | assert(rcOffsets[0] == 0 && |
| 64 | "non-zero static offset is invalid for rank-0 source memref" ); |
| 65 | return SmallVector<int64_t>{}; |
| 66 | } |
| 67 | |
| 68 | SmallVector<int64_t> offsetIdxs(srcType.getRank(), 0); |
| 69 | int64_t remainder = rcOffsets[0]; |
| 70 | SmallVector<int64_t> srcStrides = computeStrides(sizes: srcType.getShape()); |
| 71 | // Convert the scalar reinterpret_cast offset to per-dimension source starting |
| 72 | // indices. |
| 73 | for (auto [dim, stride] : llvm::enumerate(First&: srcStrides)) { |
| 74 | offsetIdxs[dim] = remainder / stride; |
| 75 | assert(offsetIdxs[dim] < srcType.getDimSize(dim) && |
| 76 | "static reinterpret_cast offset must delinearize to in-bounds " |
| 77 | "source indices" ); |
| 78 | remainder %= stride; |
| 79 | } |
| 80 | |
| 81 | assert(remainder == 0 && |
| 82 | "Assuming identity source layout, the trailing stride == 1 " |
| 83 | "so, the remainder should be 0 at the end of index calculation." ); |
| 84 | return offsetIdxs; |
| 85 | } |
| 86 | |
| 87 | static bool hasExactlyOneTruncatedNonUnitDim(memref::ReinterpretCastOp rc) { |
| 88 | MemRefType srcType = dyn_cast<MemRefType>(Val: rc.getSource().getType()); |
| 89 | MemRefType resType = dyn_cast<MemRefType>(Val: rc.getType()); |
| 90 | assert(srcType.hasStaticShape() && resType.hasStaticShape() && |
| 91 | "expected static shapes" ); |
| 92 | assert(srcType.getRank() == resType.getRank() && |
| 93 | "expected rank-preserving reinterpret_cast" ); |
| 94 | |
| 95 | unsigned truncatedDims = 0; |
| 96 | |
| 97 | for (auto [srcSize, resSize] : |
| 98 | llvm::zip_equal(t: srcType.getShape(), u: resType.getShape())) { |
| 99 | if (srcSize == resSize) |
| 100 | continue; |
| 101 | |
| 102 | // Only one non-unit source dimension may be truncated. |
| 103 | if (srcSize != 1 && resSize < srcSize) { |
| 104 | ++truncatedDims; |
| 105 | continue; |
| 106 | } |
| 107 | |
| 108 | // The size change is not a supported single non-unit source dimension |
| 109 | // reduction. |
| 110 | return false; |
| 111 | } |
| 112 | |
| 113 | // Make sure there is only one truncated dimension. |
| 114 | return truncatedDims == 1; |
| 115 | } |
| 116 | |
| 117 | /// Returns the unique non-unit dim or nullopt if # non-unit-dims != 1. |
| 118 | static std::optional<unsigned> getSingleNonUnitDim(MemRefType type) { |
| 119 | assert(type.hasStaticShape() && "expected static shape" ); |
| 120 | ArrayRef<int64_t> shape = type.getShape(); |
| 121 | |
| 122 | // Find all non-unit dims |
| 123 | auto nonUnitDims = llvm::make_filter_range( |
| 124 | Range: llvm::enumerate(First&: shape), Pred: [](auto it) { return it.value() != 1; }); |
| 125 | |
| 126 | // Expect single non-unit dims |
| 127 | if (llvm::range_size(Range&: nonUnitDims) != 1) |
| 128 | return std::nullopt; |
| 129 | |
| 130 | // Return the index of the unique non-unit dim. |
| 131 | return (*nonUnitDims.begin()).index(); |
| 132 | } |
| 133 | |
| 134 | /// Returns reinterpret_cast's result non-unit dimensions and, for static |
| 135 | /// offsets, delinearized offset. |
| 136 | /// |
| 137 | /// Supports ranked, static-shape, rank-preserving reinterpret_casts from |
| 138 | /// identity-layout sources. |
| 139 | /// * Scalar-shaped results may have arbitrary result strides. |
| 140 | /// * Non-scalar results must have static offsets, static result strides |
| 141 | /// identical to the source identity strides, and exactly one non-unit |
| 142 | /// source dimension size truncated. |
| 143 | /// |
| 144 | /// Returns nullopt for unsupported reinterpret_casts. |
| 145 | /// |
| 146 | /// Examples that return info: |
| 147 | /// |
| 148 | /// reinterpret_cast memref<1xMxNxf32, identity-layout> |
| 149 | /// to memref<1xMxKxf32, strided<[M*N, N, 1], offset: OFF>> |
| 150 | /// where K < N |
| 151 | /// |
| 152 | /// reinterpret_cast memref<1xMxf32, identity-layout> |
| 153 | /// to memref<1x1xf32, strided<[?, ?], offset: ?>> |
| 154 | /// |
| 155 | /// Examples that return no info: |
| 156 | /// |
| 157 | /// reinterpret_cast memref<1xMxNxf32, identity-layout> |
| 158 | /// to memref<1xMxKxf32, strided<[?, N, 1]>> |
| 159 | /// |
| 160 | /// reinterpret_cast memref<1xMxNxf32, identity-layout> |
| 161 | /// to memref<1xNxPxf32, strided<[M*N, N, 1], offset: OFF>> |
| 162 | /// where M != N && N != P |
| 163 | static std::optional<ResultNonUnitDimsAndOffsetsForRC> |
| 164 | getResultNonUnitDimsAndOffsetsForRC(memref::ReinterpretCastOp rc) { |
| 165 | MemRefType srcType = dyn_cast<MemRefType>(Val: rc.getSource().getType()); |
| 166 | MemRefType resType = dyn_cast<MemRefType>(Val: rc.getType()); |
| 167 | |
| 168 | // Ranked memref types are required to statically build load/store index |
| 169 | // lists. |
| 170 | if (!srcType || !resType) |
| 171 | return std::nullopt; |
| 172 | |
| 173 | // TODO: Support rank-modifying reinterpret_casts. |
| 174 | if (srcType.getRank() != resType.getRank()) |
| 175 | return std::nullopt; |
| 176 | |
| 177 | // TODO: Support dynamic shapes with mixed size operands as loop bounds. |
| 178 | if (!(srcType.hasStaticShape() && resType.hasStaticShape())) |
| 179 | return std::nullopt; |
| 180 | |
| 181 | // TODO: Support non-identity source layouts by computing source strides from |
| 182 | // the layout map. |
| 183 | if (!srcType.getLayout().isIdentity()) |
| 184 | return std::nullopt; |
| 185 | |
| 186 | ResultNonUnitDimsAndOffsetsForRC dimsAndOffs; |
| 187 | |
| 188 | // Track non-unit result dimensions; unit dimensions are |
| 189 | // always indexed at 0. |
| 190 | for (auto [dim, resultSize] : llvm::enumerate(First: resType.getShape())) { |
| 191 | if (resultSize != 1) |
| 192 | dimsAndOffs.nonUnitDimsPos.push_back(Elt: static_cast<unsigned>(dim)); |
| 193 | } |
| 194 | |
| 195 | ArrayRef<int64_t> rcOffsets = rc.getStaticOffsets(); |
| 196 | // FIXME: Despite what `getStaticOffsets` implies, `reinterpret_cast` takes |
| 197 | // only a single offset. That should be fixed at the op definition level. |
| 198 | assert(rcOffsets.size() == 1 && "Expecting single offset" ); |
| 199 | |
| 200 | bool isScalarRes = |
| 201 | llvm::all_of(Range: resType.getShape(), P: [](int64_t size) { return size == 1; }); |
| 202 | |
| 203 | bool isOffsetDynamic = ShapedType::isDynamic(dValue: rcOffsets[0]); |
| 204 | |
| 205 | // Cases with at least one non-unit dimension in reinterpret_cast's result are |
| 206 | // restricted to preserving all but one dimension from the source, which |
| 207 | // collapsed to `1` in the result, and fully static metadata. |
| 208 | if (!isScalarRes) { |
| 209 | if (isOffsetDynamic) |
| 210 | return std::nullopt; |
| 211 | |
| 212 | SmallVector<int64_t> srcIdentityStrides = |
| 213 | computeStrides(sizes: srcType.getShape()); |
| 214 | ArrayRef<int64_t> rcResultStrides = rc.getStaticStrides(); |
| 215 | |
| 216 | if (!llvm::all_of(Range: llvm::zip_equal(t&: srcIdentityStrides, u&: rcResultStrides), |
| 217 | P: [](auto pair) { |
| 218 | auto [srcStride, resultStride] = pair; |
| 219 | return !ShapedType::isDynamic(dValue: resultStride) && |
| 220 | srcStride == resultStride; |
| 221 | })) |
| 222 | return std::nullopt; |
| 223 | |
| 224 | if (!hasExactlyOneTruncatedNonUnitDim(rc)) |
| 225 | return std::nullopt; |
| 226 | } |
| 227 | |
| 228 | // CASE 1: Dynamic ReinterpretCast offset. |
| 229 | // |
| 230 | // Dynamic offsets are supported only for effectively-1D to scalar |
| 231 | // reinterpret_casts. |
| 232 | if (isOffsetDynamic) { |
| 233 | // With an effectively-1D source, a dynamic offset can be mapped to its |
| 234 | // unique non-unit dim. For other cases, bail out. |
| 235 | if (llvm::count_if(Range: srcType.getShape(), |
| 236 | P: [](int64_t size) { return size != 1; }) != 1) |
| 237 | return std::nullopt; |
| 238 | |
| 239 | return dimsAndOffs; |
| 240 | } |
| 241 | |
| 242 | // CASE 2: Static ReinterpretCast offset |
| 243 | // Delinearize static ReinterpretCast offset as in-bounds indices (one for |
| 244 | // every source dimension). |
| 245 | dimsAndOffs.delinearizedOffsets = delinearizeStaticRCOffset(rc); |
| 246 | |
| 247 | return dimsAndOffs; |
| 248 | } |
| 249 | |
| 250 | /// Rewrites supported copy operations through `memref.reinterpret_cast` to |
| 251 | /// scalar load/store operations. |
| 252 | /// |
| 253 | /// Supported cases: |
| 254 | /// 1. Scalar-shaped reinterpret_cast results. Result strides are ignored; |
| 255 | /// the store index is derived from the reinterpret_cast offset. |
| 256 | /// |
| 257 | /// 2. Non-scalar reinterpret_cast results that preserve all non-unit source |
| 258 | /// dimensions sizes except one. Result |
| 259 | /// strides must be static and identical to the identity strides of the |
| 260 | /// source, and the offset must be static. |
| 261 | /// |
| 262 | /// // BEFORE (scalar-shaped result) |
| 263 | /// %strided = memref.reinterpret_cast %dst |
| 264 | /// to offset: [OFF], sizes: [1, ..., 1], strides: [...] |
| 265 | /// memref.copy %src, %strided |
| 266 | /// |
| 267 | /// // AFTER |
| 268 | /// %v = memref.load %src[0, ..., 0] |
| 269 | /// memref.store %v, %dst[delinearized(OFF)] |
| 270 | /// |
| 271 | /// // BEFORE (one truncated non-unit dimension) |
| 272 | /// %strided = memref.reinterpret_cast %dst |
| 273 | /// to offset: [OFF], sizes: [1, M, K], strides: [M*N, N, 1] |
| 274 | /// : memref<1xMxNxf32> |
| 275 | /// to memref<1xMxKxf32, strided<[M*N, N, 1], offset: OFF>> |
| 276 | /// memref.copy %src, %strided |
| 277 | /// |
| 278 | /// // AFTER |
| 279 | /// // Assuming OFF delinearizes to [0, 0, DELIN_OFF]: |
| 280 | /// scf.for %i = 0 to M step 1 { |
| 281 | /// scf.for %k = 0 to K step 1 { |
| 282 | /// %v = memref.load %src[0, %i, %k] |
| 283 | /// memref.store %v, %dst[0, %i, DELIN_OFF + %k] |
| 284 | /// } |
| 285 | /// } |
| 286 | struct CopyToLoadAndStore : public OpRewritePattern<memref::CopyOp> { |
| 287 | public: |
| 288 | using Base::Base; |
| 289 | |
| 290 | LogicalResult matchAndRewrite(memref::CopyOp op, |
| 291 | PatternRewriter &rewriter) const final { |
| 292 | Value src = op.getSource(); |
| 293 | MemRefType cpSrcType = cast<MemRefType>(Val: src.getType()); |
| 294 | if (!cpSrcType || !cpSrcType.hasStaticShape()) |
| 295 | return rewriter.notifyMatchFailure( |
| 296 | arg&: op, msg: "only ranked, static copy sources are supported." ); |
| 297 | |
| 298 | Value rcOutput = op.getTarget(); |
| 299 | auto rc = rcOutput.getDefiningOp<memref::ReinterpretCastOp>(); |
| 300 | if (!rc) |
| 301 | return rewriter.notifyMatchFailure( |
| 302 | arg&: op, msg: "target is not a memref.reinterpret_cast" ); |
| 303 | |
| 304 | std::optional<ResultNonUnitDimsAndOffsetsForRC> dimsAndOffs = |
| 305 | getResultNonUnitDimsAndOffsetsForRC(rc); |
| 306 | if (!dimsAndOffs) |
| 307 | return rewriter.notifyMatchFailure( |
| 308 | arg&: op, |
| 309 | msg: "unsupported reinterpret_cast result dimensions, strides, or offset" ); |
| 310 | |
| 311 | Location loc = op.getLoc(); |
| 312 | Value dst = rc.getSource(); |
| 313 | MemRefType dstType = cast<MemRefType>(Val: dst.getType()); |
| 314 | MemRefType rcResType = cast<MemRefType>(Val: rc.getType()); |
| 315 | |
| 316 | // Sanity check that the copy doesn't access strided MemRef out-of-bounds. |
| 317 | // Such cases should probably be rejected by Op verifier. |
| 318 | // FIXME: Add run-time verification for cases like this. |
| 319 | if (ShapedType::isStatic(dValue: rc.getStaticOffsets()[0]) && |
| 320 | llvm::any_of(Range: llvm::enumerate(First: rcResType.getShape()), P: [&](auto it) { |
| 321 | unsigned dim = it.index(); |
| 322 | int64_t rcResultSize = it.value(); |
| 323 | return (*dimsAndOffs->delinearizedOffsets)[dim] + rcResultSize > |
| 324 | dstType.getDimSize(idx: dim); |
| 325 | })) |
| 326 | return rewriter.notifyMatchFailure(arg&: op, msg: "copy accesses are OOB" ); |
| 327 | |
| 328 | // Constant Op cache to reuse common index constants across bounds, steps, |
| 329 | // and static offsets: 0 is stored at index 0 and 1 is stored at index 1. |
| 330 | std::array<Value, 2> cachedIndexConstants; |
| 331 | auto getOrCreateIndexConstant = [&](int64_t value) -> Value { |
| 332 | if (value == 0 || value == 1) { |
| 333 | Value &cached = cachedIndexConstants[value]; |
| 334 | if (!cached) |
| 335 | cached = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value); |
| 336 | return cached; |
| 337 | } |
| 338 | return arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value); |
| 339 | }; |
| 340 | |
| 341 | auto getZeroIdxs = [&](int64_t rank) { |
| 342 | SmallVector<Value> idxs; |
| 343 | idxs.reserve(N: rank); |
| 344 | if (rank != 0) |
| 345 | idxs.append(NumInputs: rank, Elt: getOrCreateIndexConstant(0)); |
| 346 | return idxs; |
| 347 | }; |
| 348 | |
| 349 | // Create loop bounds before moving the insertion point into the loop nest, |
| 350 | // so loop-invariant constants are emitted outside the generated loops. |
| 351 | SmallVector<Value> upperBounds; |
| 352 | upperBounds.reserve(N: dimsAndOffs->nonUnitDimsPos.size()); |
| 353 | for (unsigned dim : dimsAndOffs->nonUnitDimsPos) { |
| 354 | upperBounds.push_back( |
| 355 | Elt: getOrCreateIndexConstant(rcResType.getDimSize(idx: dim))); |
| 356 | } |
| 357 | |
| 358 | // All indices are initialised to zero. |
| 359 | SmallVector<Value> rcSrcStoreIdxs = getZeroIdxs(dstType.getRank()); |
| 360 | std::optional<unsigned> srcNonUnitDimPos; |
| 361 | if (dimsAndOffs->delinearizedOffsets) { |
| 362 | // Initialize store indices from the static reinterpret_cast offset, |
| 363 | // delinearized in function gating rewrite. |
| 364 | for (auto [idx, offset] : |
| 365 | llvm::enumerate(First&: *dimsAndOffs->delinearizedOffsets)) { |
| 366 | if (offset == 0) |
| 367 | continue; |
| 368 | rcSrcStoreIdxs[idx] = getOrCreateIndexConstant(offset); |
| 369 | } |
| 370 | } else { |
| 371 | // Dynamic offset is used directly only for effectively-1D sources. |
| 372 | assert(dimsAndOffs->nonUnitDimsPos.size() <= 1 && |
| 373 | "Expecting at most one non-unit result dimension." ); |
| 374 | |
| 375 | srcNonUnitDimPos = getSingleNonUnitDim(type: dstType); |
| 376 | assert(srcNonUnitDimPos && |
| 377 | "Expecting single non-unit dimension source to receive the " |
| 378 | "dynamic offset." ); |
| 379 | |
| 380 | SmallVector<OpFoldResult> rcOffsets = rc.getMixedOffsets(); |
| 381 | // FIXME: Despite what `getMixedOffsets` implies, `reinterpret_cast` takes |
| 382 | // only a single offset. That should be fixed at the op definition level. |
| 383 | assert(rcOffsets.size() == 1 && "Expecting single offset" ); |
| 384 | // Only the index corresponding to the single non-unit dim is updated. |
| 385 | rcSrcStoreIdxs[*srcNonUnitDimPos] = |
| 386 | getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: rcOffsets[0]); |
| 387 | } |
| 388 | |
| 389 | // Create the loop nest and emit the load/store at the innermost insertion |
| 390 | // point. |
| 391 | { |
| 392 | OpBuilder::InsertionGuard guard(rewriter); |
| 393 | |
| 394 | SmallVector<Value> loadIdxs = getZeroIdxs(cpSrcType.getRank()); |
| 395 | SmallVector<Value> storeIdxs(rcSrcStoreIdxs); |
| 396 | |
| 397 | if (!dimsAndOffs->nonUnitDimsPos.empty()) { |
| 398 | Value lowerBound = getOrCreateIndexConstant(0); |
| 399 | Value step = getOrCreateIndexConstant(1); |
| 400 | |
| 401 | // Build one nested loop per non-unit reinterpret_cast result dimension. |
| 402 | for (auto [loopIndex, dim] : |
| 403 | llvm::enumerate(First&: dimsAndOffs->nonUnitDimsPos)) { |
| 404 | scf::ForOp loop = scf::ForOp::create(builder&: rewriter, location: loc, lowerBound, |
| 405 | upperBound: upperBounds[loopIndex], step); |
| 406 | |
| 407 | rewriter.setInsertionPointToStart(loop.getBody()); |
| 408 | |
| 409 | Value iv = loop.getInductionVar(); |
| 410 | // Since result strides match source identity strides dimension-wise, |
| 411 | // each IV indexes the same dimension in both the copy source and rc |
| 412 | // source. |
| 413 | loadIdxs[dim] = iv; |
| 414 | |
| 415 | if (storeIdxs[dim] == getOrCreateIndexConstant(0)) { |
| 416 | storeIdxs[dim] = iv; |
| 417 | } else { |
| 418 | storeIdxs[dim] = |
| 419 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: storeIdxs[dim], rhs: iv); |
| 420 | } |
| 421 | } |
| 422 | } |
| 423 | |
| 424 | // Emit the scalar load/store at the innermost loop body, or directly at |
| 425 | // the original copy location for scalar copies. |
| 426 | Value val = memref::LoadOp::create(builder&: rewriter, location: loc, memref: src, indices: loadIdxs); |
| 427 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: val, memref: dst, indices: storeIdxs); |
| 428 | } |
| 429 | |
| 430 | // If the only user of `rc` is the current Op (which is about to be erased), |
| 431 | // we can safely erase it. |
| 432 | bool eraseRc = rcOutput.hasOneUse(); |
| 433 | rewriter.eraseOp(op); |
| 434 | if (eraseRc) |
| 435 | rewriter.eraseOp(op: rc); |
| 436 | return success(); |
| 437 | } |
| 438 | }; |
| 439 | |
| 440 | //===----------------------------------------------------------------------===// |
| 441 | // Load Rewrite Helpers |
| 442 | //===----------------------------------------------------------------------===// |
| 443 | |
| 444 | static bool hasStaticZeroOffset(memref::ReinterpretCastOp rc) { |
| 445 | ArrayRef<int64_t> offsets = rc.getStaticOffsets(); |
| 446 | // FIXME: Despite what `getStaticOffsets` implies, `reinterpret_cast` takes |
| 447 | // only a single offset. That should be fixed at the op definition level. |
| 448 | assert(offsets.size() == 1 && "Expecting single offset" ); |
| 449 | return !ShapedType::isDynamic(dValue: offsets[0]) && offsets[0] == 0; |
| 450 | } |
| 451 | |
| 452 | static std::optional<int64_t> getConstantIndex(Value v) { |
| 453 | if (auto cst = v.getDefiningOp<arith::ConstantIndexOp>()) |
| 454 | return cst.value(); |
| 455 | // Non-constant and dynamic indices |
| 456 | return std::nullopt; |
| 457 | } |
| 458 | |
| 459 | /// Return true if input index is in bounds, i.e. `0 <= idx < upperBound`. |
| 460 | /// Fully dynamic index values (i.e. non-constant) that cannot be analysed are |
| 461 | /// treated as in-bounds. |
| 462 | static bool isConstantIndexExplicitlyOutOfBounds(Value idx, |
| 463 | int64_t upperBound) { |
| 464 | // Only statically known `arith.constant` indices are checked here. |
| 465 | std::optional<int64_t> idxVal = getConstantIndex(v: idx); |
| 466 | return idxVal && (*idxVal < 0 || *idxVal >= upperBound); |
| 467 | } |
| 468 | |
| 469 | using NonUnitDimMapping = SmallVector<std::pair<int64_t, int64_t>>; |
| 470 | |
| 471 | /// Shape restriction accepting only unit-dim insertion/removal |
| 472 | /// reinterpret_casts. |
| 473 | /// |
| 474 | /// Examples accepted: |
| 475 | /// memref<1x1x1x108xf32> <-> memref<1x108xf32> |
| 476 | /// memref<100x1xf32> <-> memref<100x1x1xf32> |
| 477 | /// memref<1x33x40xf32> <-> memref<33x1x1x40xf32> |
| 478 | /// memref<1> <-> memref<1x1x1> |
| 479 | /// |
| 480 | /// Returns the mapping of non-unit dimensions from the source |
| 481 | /// to the result MemRef if the reinterpret_cast preserved sizes and order (no |
| 482 | /// transposition) of these dimensions. |
| 483 | static std::optional<NonUnitDimMapping> |
| 484 | getNonUnitDimMapping(memref::ReinterpretCastOp rc) { |
| 485 | auto inputTy = cast<MemRefType>(Val: rc.getSource().getType()); |
| 486 | auto outputTy = cast<MemRefType>(Val: rc.getResult().getType()); |
| 487 | |
| 488 | // Only zero, statically known offsets are accepted. Non-zero or dynamic |
| 489 | // offsets would require reasoning about storage shifts in the underlying |
| 490 | // reinterpret_cast, which this helper does not model. |
| 491 | if (!hasStaticZeroOffset(rc)) |
| 492 | return std::nullopt; |
| 493 | |
| 494 | // Dynamic sizes/strides prevent precise reasoning about the underlying |
| 495 | // reinterpret_cast, so only fully static shape metadata is accepted. |
| 496 | if (llvm::any_of(Range: rc.getStaticSizes(), P: ShapedType::isDynamic) || |
| 497 | llvm::any_of(Range: rc.getStaticStrides(), P: ShapedType::isDynamic)) |
| 498 | return std::nullopt; |
| 499 | |
| 500 | ArrayRef<int64_t> inputShape = inputTy.getShape(); |
| 501 | ArrayRef<int64_t> outputShape = outputTy.getShape(); |
| 502 | int64_t inputDim = 0; |
| 503 | int64_t outputDim = 0; |
| 504 | int64_t inputRank = inputTy.getRank(); |
| 505 | int64_t outputRank = outputTy.getRank(); |
| 506 | NonUnitDimMapping mapping; |
| 507 | |
| 508 | // The preserved non-unit dimensions must have the same static sizes and |
| 509 | // appear in the same order. |
| 510 | while (inputDim < inputRank || outputDim < outputRank) { |
| 511 | if (inputDim < inputRank && inputShape[inputDim] == 1) { |
| 512 | ++inputDim; |
| 513 | continue; |
| 514 | } |
| 515 | if (outputDim < outputRank && outputShape[outputDim] == 1) { |
| 516 | ++outputDim; |
| 517 | continue; |
| 518 | } |
| 519 | |
| 520 | if (inputDim == inputRank || outputDim == outputRank) |
| 521 | return std::nullopt; |
| 522 | |
| 523 | if (ShapedType::isDynamic(dValue: inputShape[inputDim]) || |
| 524 | ShapedType::isDynamic(dValue: outputShape[outputDim]) || |
| 525 | inputShape[inputDim] != outputShape[outputDim]) |
| 526 | return std::nullopt; |
| 527 | |
| 528 | mapping.push_back(Elt: {inputDim, outputDim}); |
| 529 | ++inputDim; |
| 530 | ++outputDim; |
| 531 | } |
| 532 | return mapping; |
| 533 | } |
| 534 | |
| 535 | /// Checks statically known and constant indices accessed by a load from a |
| 536 | /// unit-dim insertion/removal reinterpret_cast to ensure in-bounds only access. |
| 537 | /// Fully dynamic indices are skipped (there is no way to verify them). |
| 538 | [[maybe_unused]] static bool areIndicesInBounds(memref::LoadOp load) { |
| 539 | auto rc = load.getMemRef().getDefiningOp<memref::ReinterpretCastOp>(); |
| 540 | auto rcOutputTy = cast<MemRefType>(Val: rc.getResult().getType()); |
| 541 | |
| 542 | for (auto [pos, idx] : llvm::enumerate(First: load.getIndices())) { |
| 543 | // FIXME: This should be ensured by the memref.load semantics. |
| 544 | // In the long term, this sanity-check may live in the same debug-only |
| 545 | // checks as `MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS`. This rejects |
| 546 | // only explicit constant OOB indices. Dynamic/non-constant indices are not |
| 547 | // filtered here. |
| 548 | if (isConstantIndexExplicitlyOutOfBounds(idx, upperBound: rcOutputTy.getDimSize(idx: pos))) |
| 549 | return false; |
| 550 | } |
| 551 | return true; |
| 552 | } |
| 553 | |
| 554 | /// Rewrites `memref.load` through a reinterpret_cast that only inserts/removes |
| 555 | /// unit dimensions by mapping the load indices directly onto the source MemRef. |
| 556 | /// |
| 557 | /// Shape restriction gated by getNonUnitDimMapping(). |
| 558 | /// |
| 559 | /// BEFORE (rank expansion) |
| 560 | /// %view = memref.reinterpret_cast %src |
| 561 | /// : memref<1xNxMxf32> to memref<Nx1x1xMxf32> |
| 562 | /// %v = memref.load %view[%i, %c0, %c0, %j] : memref<Nx1x1xMxf32> |
| 563 | /// |
| 564 | /// AFTER |
| 565 | /// %v = memref.load %src[%c0, %i, %j] : memref<1xNxMxf32> |
| 566 | /// |
| 567 | /// BEFORE (rank collapsing) |
| 568 | /// %view = memref.reinterpret_cast %src |
| 569 | /// : memref<Nx1x1xMxf32> to memref<1xNxMxf32> |
| 570 | /// %v = memref.load %view[%c0, %i, %j] : memref<1xNxMxf32> |
| 571 | /// |
| 572 | /// AFTER |
| 573 | /// %v = memref.load %src[%i, %c0, %c0, %j] : memref<Nx1x1xMxf32> |
| 574 | struct RewriteLoadFromReinterpretCast |
| 575 | : public OpRewritePattern<memref::LoadOp> { |
| 576 | public: |
| 577 | using Base::Base; |
| 578 | |
| 579 | LogicalResult matchAndRewrite(memref::LoadOp op, |
| 580 | PatternRewriter &rewriter) const override { |
| 581 | auto rc = op.getMemRef().getDefiningOp<memref::ReinterpretCastOp>(); |
| 582 | if (!rc) |
| 583 | return rewriter.notifyMatchFailure( |
| 584 | arg&: op, msg: "target is not a memref.reinterpret_cast" ); |
| 585 | std::optional<NonUnitDimMapping> dimMapping = getNonUnitDimMapping(rc); |
| 586 | if (!dimMapping) |
| 587 | return rewriter.notifyMatchFailure( |
| 588 | arg&: op, msg: "reinterpret_cast is not a unit-dim insertion/removal preserving " |
| 589 | "non-unit dimensions" ); |
| 590 | |
| 591 | assert(areIndicesInBounds(op) && |
| 592 | "load from reinterpret_cast indexes out of bounds!" ); |
| 593 | |
| 594 | auto rcInputTy = cast<MemRefType>(Val: rc.getSource().getType()); |
| 595 | |
| 596 | int64_t rcInputRank = rcInputTy.getRank(); |
| 597 | |
| 598 | SmallVector<Value> oldIdxs(op.getIndices().begin(), op.getIndices().end()); |
| 599 | |
| 600 | // Prefer reusing an explicit constant-zero index from the old load. |
| 601 | Value zeroIndex; |
| 602 | for (Value idx : oldIdxs) { |
| 603 | std::optional<int64_t> idxVal = getConstantIndex(v: idx); |
| 604 | if (idxVal && *idxVal == 0) { |
| 605 | zeroIndex = idx; |
| 606 | break; |
| 607 | } |
| 608 | } |
| 609 | if (!zeroIndex) |
| 610 | zeroIndex = arith::ConstantIndexOp::create(builder&: rewriter, location: op.getLoc(), value: 0); |
| 611 | |
| 612 | // Initialize new load indices to all 0s. |
| 613 | SmallVector<Value> rcInputIdxs(rcInputRank, zeroIndex); |
| 614 | for (auto [inputDim, outputDim] : *dimMapping) |
| 615 | rcInputIdxs[inputDim] = oldIdxs[outputDim]; |
| 616 | |
| 617 | auto rcInput = rc.getSource(); |
| 618 | // If the only user of rc is the current Op (which is about to be erased), |
| 619 | // we can safely erase it. |
| 620 | if (rc.getResult().hasOneUse()) |
| 621 | rewriter.eraseOp(op: rc); |
| 622 | rewriter.replaceOpWithNewOp<memref::LoadOp>( |
| 623 | op, args&: rcInput, args&: rcInputIdxs, args: op.getNontemporalAttr(), |
| 624 | args: op.getAlignmentAttr(), args: op.getInvariantAttr()); |
| 625 | return success(); |
| 626 | } |
| 627 | }; |
| 628 | |
| 629 | struct ElideReinterpretCastPass |
| 630 | : public memref::impl::ElideReinterpretCastPassBase< |
| 631 | ElideReinterpretCastPass> { |
| 632 | void runOnOperation() override { |
| 633 | MLIRContext &ctx = getContext(); |
| 634 | |
| 635 | RewritePatternSet patterns(&ctx); |
| 636 | memref::populateElideReinterpretCastPatterns(patterns); |
| 637 | ConversionTarget target(ctx); |
| 638 | target.addDynamicallyLegalOp<memref::CopyOp>(callback: [](memref::CopyOp op) { |
| 639 | auto rc = op.getTarget().getDefiningOp<memref::ReinterpretCastOp>(); |
| 640 | if (!rc) |
| 641 | return true; |
| 642 | // Pattern applies only when the copy source shape is static and the |
| 643 | // reinterpret_cast result can be mapped back to base memref indices. |
| 644 | MemRefType cpSrcType = dyn_cast<MemRefType>(Val: op.getSource().getType()); |
| 645 | return !(cpSrcType && cpSrcType.hasStaticShape() && |
| 646 | getResultNonUnitDimsAndOffsetsForRC(rc)); |
| 647 | }); |
| 648 | target.addDynamicallyLegalOp<memref::LoadOp>(callback: [](memref::LoadOp op) { |
| 649 | auto rc = op.getMemRef().getDefiningOp<memref::ReinterpretCastOp>(); |
| 650 | if (!rc) |
| 651 | return true; |
| 652 | return !getNonUnitDimMapping(rc); |
| 653 | }); |
| 654 | target.addLegalDialect<arith::ArithDialect, memref::MemRefDialect, |
| 655 | scf::SCFDialect>(); |
| 656 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 657 | patterns: std::move(patterns)))) |
| 658 | signalPassFailure(); |
| 659 | } |
| 660 | }; |
| 661 | |
| 662 | } // namespace |
| 663 | |
| 664 | void mlir::memref::populateElideReinterpretCastPatterns( |
| 665 | RewritePatternSet &patterns) { |
| 666 | patterns.add<CopyToLoadAndStore, RewriteLoadFromReinterpretCast>( |
| 667 | arg: patterns.getContext()); |
| 668 | } |
| 669 | |