1//=== AffineTransformOps.cpp - Implementation of Affine transformation ops ===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.h"
10#include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
11#include "mlir/Dialect/Affine/Analysis/Utils.h"
12#include "mlir/Dialect/Affine/IR/AffineOps.h"
13#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
14#include "mlir/Dialect/Affine/LoopUtils.h"
15#include "mlir/Dialect/Affine/Transforms/Transforms.h"
16#include "mlir/Dialect/Affine/Utils.h"
17#include "mlir/Dialect/Transform/IR/TransformDialect.h"
18#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
19#include "mlir/Dialect/Vector/IR/VectorDialect.h"
20#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
21#include "llvm/ADT/ArrayRef.h"
22#include <cstdint>
23
24using namespace mlir;
25using namespace mlir::affine;
26using namespace mlir::transform;
27
28//===----------------------------------------------------------------------===//
29// SimplifyBoundedAffineOpsOp
30//===----------------------------------------------------------------------===//
31
32LogicalResult SimplifyBoundedAffineOpsOp::verify() {
33 if (getLowerBounds().size() != getBoundedValues().size())
34 return emitOpError() << "incorrect number of lower bounds, expected "
35 << getBoundedValues().size() << " but found "
36 << getLowerBounds().size();
37 if (getUpperBounds().size() != getBoundedValues().size())
38 return emitOpError() << "incorrect number of upper bounds, expected "
39 << getBoundedValues().size() << " but found "
40 << getUpperBounds().size();
41 return success();
42}
43
44namespace {
45/// Simplify affine.min / affine.max ops with the given constraints. They are
46/// either rewritten to affine.apply or left unchanged.
47template <typename OpTy>
48struct SimplifyAffineMinMaxOp : public OpRewritePattern<OpTy> {
49 using OpRewritePattern<OpTy>::OpRewritePattern;
50 SimplifyAffineMinMaxOp(MLIRContext *ctx,
51 const FlatAffineValueConstraints &constraints,
52 PatternBenefit benefit = 1)
53 : OpRewritePattern<OpTy>(ctx, benefit), constraints(constraints) {}
54
55 LogicalResult matchAndRewrite(OpTy op,
56 PatternRewriter &rewriter) const override {
57 FailureOr<AffineValueMap> simplified =
58 simplifyConstrainedMinMaxOp(op, constraints);
59 if (failed(Result: simplified))
60 return failure();
61 rewriter.replaceOpWithNewOp<AffineApplyOp>(op, simplified->getAffineMap(),
62 simplified->getOperands());
63 return success();
64 }
65
66 const FlatAffineValueConstraints &constraints;
67};
68} // namespace
69
70DiagnosedSilenceableFailure
71SimplifyBoundedAffineOpsOp::apply(transform::TransformRewriter &rewriter,
72 TransformResults &results,
73 TransformState &state) {
74 // Get constraints for bounded values.
75 SmallVector<int64_t> lbs;
76 SmallVector<int64_t> ubs;
77 SmallVector<Value> boundedValues;
78 DenseSet<Operation *> boundedOps;
79 for (const auto &it : llvm::zip_equal(t: getBoundedValues(), u: getLowerBounds(),
80 args: getUpperBounds())) {
81 Value handle = std::get<0>(t: it);
82 for (Operation *op : state.getPayloadOps(value: handle)) {
83 if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) {
84 auto diag =
85 emitDefiniteFailure()
86 << "expected bounded value handle to point to one or multiple "
87 "single-result index-typed ops";
88 diag.attachNote(loc: op->getLoc()) << "multiple/non-index result";
89 return diag;
90 }
91 boundedValues.push_back(Elt: op->getResult(idx: 0));
92 boundedOps.insert(V: op);
93 lbs.push_back(Elt: std::get<1>(t: it));
94 ubs.push_back(Elt: std::get<2>(t: it));
95 }
96 }
97
98 // Build constraint set.
99 FlatAffineValueConstraints cstr;
100 for (const auto &it : llvm::zip(t&: boundedValues, u&: lbs, args&: ubs)) {
101 unsigned pos;
102 if (!cstr.findVar(val: std::get<0>(t: it), pos: &pos))
103 pos = cstr.appendSymbolVar(vals: std::get<0>(t: it));
104 cstr.addBound(type: presburger::BoundType::LB, pos, value: std::get<1>(t: it));
105 // Note: addBound bounds are inclusive, but specified UB is exclusive.
106 cstr.addBound(type: presburger::BoundType::UB, pos, value: std::get<2>(t: it) - 1);
107 }
108
109 // Transform all targets.
110 SmallVector<Operation *> targets;
111 for (Operation *target : state.getPayloadOps(value: getTarget())) {
112 if (!isa<AffineMinOp, AffineMaxOp>(Val: target)) {
113 auto diag = emitDefiniteFailure()
114 << "target must be affine.min or affine.max";
115 diag.attachNote(loc: target->getLoc()) << "target op";
116 return diag;
117 }
118 if (boundedOps.contains(V: target)) {
119 auto diag = emitDefiniteFailure()
120 << "target op result must not be constrained";
121 diag.attachNote(loc: target->getLoc()) << "target/constrained op";
122 return diag;
123 }
124 targets.push_back(Elt: target);
125 }
126 RewritePatternSet patterns(getContext());
127 // Canonicalization patterns are needed so that affine.apply ops are composed
128 // with the remaining affine.min/max ops.
129 AffineMaxOp::getCanonicalizationPatterns(results&: patterns, context: getContext());
130 AffineMinOp::getCanonicalizationPatterns(results&: patterns, context: getContext());
131 patterns.insert<SimplifyAffineMinMaxOp<AffineMinOp>,
132 SimplifyAffineMinMaxOp<AffineMaxOp>>(arg: getContext(), args&: cstr);
133 FrozenRewritePatternSet frozenPatterns(std::move(patterns));
134 // Apply the simplification pattern to a fixpoint.
135 if (failed(Result: applyOpPatternsGreedily(
136 ops: targets, patterns: frozenPatterns,
137 config: GreedyRewriteConfig()
138 .setListener(
139 static_cast<RewriterBase::Listener *>(rewriter.getListener()))
140 .setStrictness(GreedyRewriteStrictness::ExistingAndNewOps)))) {
141 auto diag = emitDefiniteFailure()
142 << "affine.min/max simplification did not converge";
143 return diag;
144 }
145 return DiagnosedSilenceableFailure::success();
146}
147
148void SimplifyBoundedAffineOpsOp::getEffects(
149 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
150 consumesHandle(handles: getTargetMutable(), effects);
151 for (OpOperand &operand : getBoundedValuesMutable())
152 onlyReadsHandle(handles: operand, effects);
153 modifiesPayload(effects);
154}
155
156//===----------------------------------------------------------------------===//
157// SimplifyMinMaxAffineOpsOp
158//===----------------------------------------------------------------------===//
159
160LogicalResult SuperVectorizeOp::verify() {
161 if (getFastestVaryingPattern().has_value()) {
162 if (getFastestVaryingPattern()->size() != getVectorSizes().size())
163 return emitOpError()
164 << "fastest varying pattern specified with different size than "
165 "the vector size";
166 }
167 return success();
168}
169
170DiagnosedSilenceableFailure
171SuperVectorizeOp::apply(transform::TransformRewriter &rewriter,
172 TransformResults &results, TransformState &state) {
173 ArrayRef<int64_t> fastestVaryingPattern;
174 if (getFastestVaryingPattern().has_value())
175 fastestVaryingPattern = getFastestVaryingPattern().value();
176
177 for (Operation *target : state.getPayloadOps(value: getTarget()))
178 if (!target->getParentOfType<affine::AffineForOp>())
179 vectorizeChildAffineLoops(parentOp: target, vectorizeReductions: getVectorizeReductions(),
180 vectorSizes: getVectorSizes(), fastestVaryingPattern);
181
182 return DiagnosedSilenceableFailure::success();
183}
184
185void SuperVectorizeOp::getEffects(
186 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
187 consumesHandle(handles: getTargetMutable(), effects);
188 modifiesPayload(effects);
189}
190
191//===----------------------------------------------------------------------===//
192// SimplifyMinMaxAffineOpsOp
193//===----------------------------------------------------------------------===//
194DiagnosedSilenceableFailure
195SimplifyMinMaxAffineOpsOp::apply(transform::TransformRewriter &rewriter,
196 TransformResults &results,
197 TransformState &state) {
198 SmallVector<Operation *> targets;
199 for (Operation *target : state.getPayloadOps(value: getTarget())) {
200 if (!isa<AffineMinOp, AffineMaxOp>(Val: target)) {
201 auto diag = emitDefiniteFailure()
202 << "target must be affine.min or affine.max";
203 diag.attachNote(loc: target->getLoc()) << "target op";
204 return diag;
205 }
206 targets.push_back(Elt: target);
207 }
208 bool modified = false;
209 if (failed(Result: mlir::affine::simplifyAffineMinMaxOps(rewriter, ops: targets,
210 modified: &modified))) {
211 return emitDefiniteFailure()
212 << "affine.min/max simplification did not converge";
213 }
214 if (!modified) {
215 return emitSilenceableError()
216 << "the transform failed to simplify any of the target operations";
217 }
218 return DiagnosedSilenceableFailure::success();
219}
220
221void SimplifyMinMaxAffineOpsOp::getEffects(
222 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
223 consumesHandle(handles: getTargetMutable(), effects);
224 modifiesPayload(effects);
225}
226
227//===----------------------------------------------------------------------===//
228// Transform op registration
229//===----------------------------------------------------------------------===//
230
231namespace {
232class AffineTransformDialectExtension
233 : public transform::TransformDialectExtension<
234 AffineTransformDialectExtension> {
235public:
236 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AffineTransformDialectExtension)
237
238 using Base::Base;
239
240 void init() {
241 declareGeneratedDialect<AffineDialect>();
242 declareGeneratedDialect<vector::VectorDialect>();
243
244 registerTransformOps<
245#define GET_OP_LIST
246#include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.cpp.inc"
247 >();
248 }
249};
250} // namespace
251
252#define GET_OP_CLASSES
253#include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.cpp.inc"
254
255void mlir::affine::registerTransformDialectExtension(
256 DialectRegistry &registry) {
257 registry.addExtensions<AffineTransformDialectExtension>();
258}
259