1//===- AffineExpandIndexOpsAsAffine.cpp - Expand index ops to apply pass --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to expand affine index ops into one or more more
10// fundamental operations.
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Affine/Transforms/Passes.h"
14
15#include "mlir/Dialect/Affine/IR/AffineOps.h"
16#include "mlir/Dialect/Affine/Transforms/Transforms.h"
17#include "mlir/Dialect/Affine/Utils.h"
18#include "mlir/Dialect/Arith/Utils/Utils.h"
19#include "mlir/Dialect/UB/IR/UBOps.h"
20#include "mlir/Dialect/Utils/IndexingUtils.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24namespace mlir {
25namespace affine {
26#define GEN_PASS_DEF_AFFINEEXPANDINDEXOPSASAFFINE
27#include "mlir/Dialect/Affine/Transforms/Passes.h.inc"
28} // namespace affine
29} // namespace mlir
30
31using namespace mlir;
32using namespace mlir::affine;
33
34namespace {
35/// Lowers `affine.delinearize_index` into a sequence of division and remainder
36/// operations via affine.apply. For vector types, unrolls to per-element
37/// scalar affine.apply operations.
38struct LowerDelinearizeIndexOps
39 : public OpRewritePattern<AffineDelinearizeIndexOp> {
40 using OpRewritePattern<AffineDelinearizeIndexOp>::OpRewritePattern;
41 LogicalResult matchAndRewrite(AffineDelinearizeIndexOp op,
42 PatternRewriter &rewriter) const override {
43 Location loc = op.getLoc();
44 Value linearIndex = op.getLinearIndex();
45 auto vecTy = dyn_cast<VectorType>(Val: linearIndex.getType());
46
47 // Scalar case: use the existing affine lowering path.
48 if (!vecTy) {
49 FailureOr<SmallVector<Value>> multiIndex =
50 delinearizeIndex(b&: rewriter, loc, linearIndex, basis: op.getEffectiveBasis(),
51 /*hasOuterBound=*/false);
52 if (failed(Result: multiIndex))
53 return failure();
54 rewriter.replaceOp(op, newValues: *multiIndex);
55 return success();
56 }
57
58 // Vector case: unroll to per-element scalar affine.apply operations
59 // using StaticTileOffsetRange for multi-dimensional vector support.
60 if (vecTy.isScalable())
61 return rewriter.notifyMatchFailure(arg&: op, msg: "scalable vectors not supported");
62
63 unsigned numResults = op.getNumResults();
64 ArrayRef<int64_t> shape = vecTy.getShape();
65 SmallVector<int64_t> tileShape(shape.size(), 1);
66
67 SmallVector<Value> resultVecs(numResults);
68 Value poison = ub::PoisonOp::create(builder&: rewriter, location: loc, result: vecTy);
69 for (unsigned r = 0; r < numResults; ++r)
70 resultVecs[r] = poison;
71
72 for (SmallVector<int64_t> pos : StaticTileOffsetRange(shape, tileShape)) {
73 Value scalar = vector::ExtractOp::create(builder&: rewriter, location: loc, source: linearIndex, position: pos);
74
75 FailureOr<SmallVector<Value>> scalarResults =
76 delinearizeIndex(b&: rewriter, loc, linearIndex: scalar, basis: op.getEffectiveBasis(),
77 /*hasOuterBound=*/false);
78 if (failed(Result: scalarResults))
79 return failure();
80
81 for (unsigned r = 0; r < numResults; ++r)
82 resultVecs[r] = vector::InsertOp::create(
83 builder&: rewriter, location: loc, valueToStore: (*scalarResults)[r], dest: resultVecs[r], position: pos);
84 }
85
86 rewriter.replaceOp(op, newValues: resultVecs);
87 return success();
88 }
89};
90
91/// Lowers `affine.linearize_index` into a sequence of multiplications and
92/// additions via affine.apply. For vector types, unrolls to per-element
93/// scalar affine.apply operations.
94struct LowerLinearizeIndexOps final : OpRewritePattern<AffineLinearizeIndexOp> {
95 using Base::Base;
96 LogicalResult matchAndRewrite(AffineLinearizeIndexOp op,
97 PatternRewriter &rewriter) const override {
98 Location loc = op.getLoc();
99 auto vecTy = dyn_cast<VectorType>(Val: op.getLinearIndex().getType());
100
101 // Scalar case: use the existing affine lowering path.
102 if (!vecTy) {
103 // Should be folded away, included here for safety.
104 if (op.getMultiIndex().empty()) {
105 rewriter.replaceOpWithNewOp<arith::ConstantIndexOp>(op, args: 0);
106 return success();
107 }
108
109 SmallVector<OpFoldResult> multiIndex =
110 getAsOpFoldResult(values: op.getMultiIndex());
111 OpFoldResult linearIndex =
112 linearizeIndex(builder&: rewriter, loc, multiIndex, basis: op.getMixedBasis());
113 Value linearIndexValue =
114 getValueOrCreateConstantIntOp(b&: rewriter, loc, ofr: linearIndex);
115 rewriter.replaceOp(op, newValues: linearIndexValue);
116 return success();
117 }
118
119 // Vector case: unroll to per-element scalar affine.apply operations
120 // using StaticTileOffsetRange for multi-dimensional vector support.
121 if (vecTy.isScalable())
122 return rewriter.notifyMatchFailure(arg&: op, msg: "scalable vectors not supported");
123
124 ArrayRef<int64_t> shape = vecTy.getShape();
125 SmallVector<int64_t> tileShape(shape.size(), 1);
126 ValueRange multiIndex = op.getMultiIndex();
127
128 Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: vecTy);
129
130 for (SmallVector<int64_t> pos : StaticTileOffsetRange(shape, tileShape)) {
131 SmallVector<OpFoldResult> scalarIndices;
132 for (Value vec : multiIndex)
133 scalarIndices.push_back(
134 Elt: vector::ExtractOp::create(builder&: rewriter, location: loc, source: vec, position: pos).getResult());
135
136 OpFoldResult linearIndex =
137 linearizeIndex(builder&: rewriter, loc, multiIndex: scalarIndices, basis: op.getMixedBasis());
138 Value scalarResult =
139 getValueOrCreateConstantIntOp(b&: rewriter, loc, ofr: linearIndex);
140
141 result =
142 vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: scalarResult, dest: result, position: pos);
143 }
144
145 rewriter.replaceOp(op, newValues: result);
146 return success();
147 }
148};
149
150class ExpandAffineIndexOpsAsAffinePass
151 : public affine::impl::AffineExpandIndexOpsAsAffineBase<
152 ExpandAffineIndexOpsAsAffinePass> {
153public:
154 ExpandAffineIndexOpsAsAffinePass() = default;
155
156 void runOnOperation() override {
157 MLIRContext *context = &getContext();
158 RewritePatternSet patterns(context);
159 populateAffineExpandIndexOpsAsAffinePatterns(patterns);
160 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
161 return signalPassFailure();
162 }
163};
164
165} // namespace
166
167void mlir::affine::populateAffineExpandIndexOpsAsAffinePatterns(
168 RewritePatternSet &patterns) {
169 patterns.insert<LowerDelinearizeIndexOps, LowerLinearizeIndexOps>(
170 arg: patterns.getContext());
171}
172
173std::unique_ptr<Pass> mlir::affine::createAffineExpandIndexOpsAsAffinePass() {
174 return std::make_unique<ExpandAffineIndexOpsAsAffinePass>();
175}
176