| 1 | //=== AffineTransformOps.cpp - Implementation of Affine transformation ops ===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.h" |
| 10 | #include "mlir/Dialect/Affine/Analysis/AffineStructures.h" |
| 11 | #include "mlir/Dialect/Affine/Analysis/Utils.h" |
| 12 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 13 | #include "mlir/Dialect/Affine/IR/AffineValueMap.h" |
| 14 | #include "mlir/Dialect/Affine/LoopUtils.h" |
| 15 | #include "mlir/Dialect/Affine/Transforms/Transforms.h" |
| 16 | #include "mlir/Dialect/Affine/Utils.h" |
| 17 | #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
| 18 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
| 19 | #include "mlir/Dialect/Vector/IR/VectorDialect.h" |
| 20 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 21 | #include "llvm/ADT/ArrayRef.h" |
| 22 | #include <cstdint> |
| 23 | |
| 24 | using namespace mlir; |
| 25 | using namespace mlir::affine; |
| 26 | using namespace mlir::transform; |
| 27 | |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | // SimplifyBoundedAffineOpsOp |
| 30 | //===----------------------------------------------------------------------===// |
| 31 | |
| 32 | LogicalResult SimplifyBoundedAffineOpsOp::verify() { |
| 33 | if (getLowerBounds().size() != getBoundedValues().size()) |
| 34 | return emitOpError() << "incorrect number of lower bounds, expected " |
| 35 | << getBoundedValues().size() << " but found " |
| 36 | << getLowerBounds().size(); |
| 37 | if (getUpperBounds().size() != getBoundedValues().size()) |
| 38 | return emitOpError() << "incorrect number of upper bounds, expected " |
| 39 | << getBoundedValues().size() << " but found " |
| 40 | << getUpperBounds().size(); |
| 41 | return success(); |
| 42 | } |
| 43 | |
| 44 | namespace { |
| 45 | /// Simplify affine.min / affine.max ops with the given constraints. They are |
| 46 | /// either rewritten to affine.apply or left unchanged. |
| 47 | template <typename OpTy> |
| 48 | struct SimplifyAffineMinMaxOp : public OpRewritePattern<OpTy> { |
| 49 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 50 | SimplifyAffineMinMaxOp(MLIRContext *ctx, |
| 51 | const FlatAffineValueConstraints &constraints, |
| 52 | PatternBenefit benefit = 1) |
| 53 | : OpRewritePattern<OpTy>(ctx, benefit), constraints(constraints) {} |
| 54 | |
| 55 | LogicalResult matchAndRewrite(OpTy op, |
| 56 | PatternRewriter &rewriter) const override { |
| 57 | FailureOr<AffineValueMap> simplified = |
| 58 | simplifyConstrainedMinMaxOp(op, constraints); |
| 59 | if (failed(Result: simplified)) |
| 60 | return failure(); |
| 61 | rewriter.replaceOpWithNewOp<AffineApplyOp>(op, simplified->getAffineMap(), |
| 62 | simplified->getOperands()); |
| 63 | return success(); |
| 64 | } |
| 65 | |
| 66 | const FlatAffineValueConstraints &constraints; |
| 67 | }; |
| 68 | } // namespace |
| 69 | |
| 70 | DiagnosedSilenceableFailure |
| 71 | SimplifyBoundedAffineOpsOp::apply(transform::TransformRewriter &rewriter, |
| 72 | TransformResults &results, |
| 73 | TransformState &state) { |
| 74 | // Get constraints for bounded values. |
| 75 | SmallVector<int64_t> lbs; |
| 76 | SmallVector<int64_t> ubs; |
| 77 | SmallVector<Value> boundedValues; |
| 78 | DenseSet<Operation *> boundedOps; |
| 79 | for (const auto &it : llvm::zip_equal(t: getBoundedValues(), u: getLowerBounds(), |
| 80 | args: getUpperBounds())) { |
| 81 | Value handle = std::get<0>(t: it); |
| 82 | for (Operation *op : state.getPayloadOps(value: handle)) { |
| 83 | if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) { |
| 84 | auto diag = |
| 85 | emitDefiniteFailure() |
| 86 | << "expected bounded value handle to point to one or multiple " |
| 87 | "single-result index-typed ops" ; |
| 88 | diag.attachNote(loc: op->getLoc()) << "multiple/non-index result" ; |
| 89 | return diag; |
| 90 | } |
| 91 | boundedValues.push_back(Elt: op->getResult(idx: 0)); |
| 92 | boundedOps.insert(V: op); |
| 93 | lbs.push_back(Elt: std::get<1>(t: it)); |
| 94 | ubs.push_back(Elt: std::get<2>(t: it)); |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | // Build constraint set. |
| 99 | FlatAffineValueConstraints cstr; |
| 100 | for (const auto &it : llvm::zip(t&: boundedValues, u&: lbs, args&: ubs)) { |
| 101 | unsigned pos; |
| 102 | if (!cstr.findVar(val: std::get<0>(t: it), pos: &pos)) |
| 103 | pos = cstr.appendSymbolVar(vals: std::get<0>(t: it)); |
| 104 | cstr.addBound(type: presburger::BoundType::LB, pos, value: std::get<1>(t: it)); |
| 105 | // Note: addBound bounds are inclusive, but specified UB is exclusive. |
| 106 | cstr.addBound(type: presburger::BoundType::UB, pos, value: std::get<2>(t: it) - 1); |
| 107 | } |
| 108 | |
| 109 | // Transform all targets. |
| 110 | SmallVector<Operation *> targets; |
| 111 | for (Operation *target : state.getPayloadOps(value: getTarget())) { |
| 112 | if (!isa<AffineMinOp, AffineMaxOp>(Val: target)) { |
| 113 | auto diag = emitDefiniteFailure() |
| 114 | << "target must be affine.min or affine.max" ; |
| 115 | diag.attachNote(loc: target->getLoc()) << "target op" ; |
| 116 | return diag; |
| 117 | } |
| 118 | if (boundedOps.contains(V: target)) { |
| 119 | auto diag = emitDefiniteFailure() |
| 120 | << "target op result must not be constrained" ; |
| 121 | diag.attachNote(loc: target->getLoc()) << "target/constrained op" ; |
| 122 | return diag; |
| 123 | } |
| 124 | targets.push_back(Elt: target); |
| 125 | } |
| 126 | RewritePatternSet patterns(getContext()); |
| 127 | // Canonicalization patterns are needed so that affine.apply ops are composed |
| 128 | // with the remaining affine.min/max ops. |
| 129 | AffineMaxOp::getCanonicalizationPatterns(results&: patterns, context: getContext()); |
| 130 | AffineMinOp::getCanonicalizationPatterns(results&: patterns, context: getContext()); |
| 131 | patterns.insert<SimplifyAffineMinMaxOp<AffineMinOp>, |
| 132 | SimplifyAffineMinMaxOp<AffineMaxOp>>(arg: getContext(), args&: cstr); |
| 133 | FrozenRewritePatternSet frozenPatterns(std::move(patterns)); |
| 134 | // Apply the simplification pattern to a fixpoint. |
| 135 | if (failed(Result: applyOpPatternsGreedily( |
| 136 | ops: targets, patterns: frozenPatterns, |
| 137 | config: GreedyRewriteConfig() |
| 138 | .setListener( |
| 139 | static_cast<RewriterBase::Listener *>(rewriter.getListener())) |
| 140 | .setStrictness(GreedyRewriteStrictness::ExistingAndNewOps)))) { |
| 141 | auto diag = emitDefiniteFailure() |
| 142 | << "affine.min/max simplification did not converge" ; |
| 143 | return diag; |
| 144 | } |
| 145 | return DiagnosedSilenceableFailure::success(); |
| 146 | } |
| 147 | |
| 148 | void SimplifyBoundedAffineOpsOp::getEffects( |
| 149 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 150 | consumesHandle(handles: getTargetMutable(), effects); |
| 151 | for (OpOperand &operand : getBoundedValuesMutable()) |
| 152 | onlyReadsHandle(handles: operand, effects); |
| 153 | modifiesPayload(effects); |
| 154 | } |
| 155 | |
| 156 | //===----------------------------------------------------------------------===// |
| 157 | // SimplifyMinMaxAffineOpsOp |
| 158 | //===----------------------------------------------------------------------===// |
| 159 | |
| 160 | LogicalResult SuperVectorizeOp::verify() { |
| 161 | if (getFastestVaryingPattern().has_value()) { |
| 162 | if (getFastestVaryingPattern()->size() != getVectorSizes().size()) |
| 163 | return emitOpError() |
| 164 | << "fastest varying pattern specified with different size than " |
| 165 | "the vector size" ; |
| 166 | } |
| 167 | return success(); |
| 168 | } |
| 169 | |
| 170 | DiagnosedSilenceableFailure |
| 171 | SuperVectorizeOp::apply(transform::TransformRewriter &rewriter, |
| 172 | TransformResults &results, TransformState &state) { |
| 173 | ArrayRef<int64_t> fastestVaryingPattern; |
| 174 | if (getFastestVaryingPattern().has_value()) |
| 175 | fastestVaryingPattern = getFastestVaryingPattern().value(); |
| 176 | |
| 177 | for (Operation *target : state.getPayloadOps(value: getTarget())) |
| 178 | if (!target->getParentOfType<affine::AffineForOp>()) |
| 179 | vectorizeChildAffineLoops(parentOp: target, vectorizeReductions: getVectorizeReductions(), |
| 180 | vectorSizes: getVectorSizes(), fastestVaryingPattern); |
| 181 | |
| 182 | return DiagnosedSilenceableFailure::success(); |
| 183 | } |
| 184 | |
| 185 | void SuperVectorizeOp::getEffects( |
| 186 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 187 | consumesHandle(handles: getTargetMutable(), effects); |
| 188 | modifiesPayload(effects); |
| 189 | } |
| 190 | |
| 191 | //===----------------------------------------------------------------------===// |
| 192 | // SimplifyMinMaxAffineOpsOp |
| 193 | //===----------------------------------------------------------------------===// |
| 194 | DiagnosedSilenceableFailure |
| 195 | SimplifyMinMaxAffineOpsOp::apply(transform::TransformRewriter &rewriter, |
| 196 | TransformResults &results, |
| 197 | TransformState &state) { |
| 198 | SmallVector<Operation *> targets; |
| 199 | for (Operation *target : state.getPayloadOps(value: getTarget())) { |
| 200 | if (!isa<AffineMinOp, AffineMaxOp>(Val: target)) { |
| 201 | auto diag = emitDefiniteFailure() |
| 202 | << "target must be affine.min or affine.max" ; |
| 203 | diag.attachNote(loc: target->getLoc()) << "target op" ; |
| 204 | return diag; |
| 205 | } |
| 206 | targets.push_back(Elt: target); |
| 207 | } |
| 208 | bool modified = false; |
| 209 | if (failed(Result: mlir::affine::simplifyAffineMinMaxOps(rewriter, ops: targets, |
| 210 | modified: &modified))) { |
| 211 | return emitDefiniteFailure() |
| 212 | << "affine.min/max simplification did not converge" ; |
| 213 | } |
| 214 | if (!modified) { |
| 215 | return emitSilenceableError() |
| 216 | << "the transform failed to simplify any of the target operations" ; |
| 217 | } |
| 218 | return DiagnosedSilenceableFailure::success(); |
| 219 | } |
| 220 | |
| 221 | void SimplifyMinMaxAffineOpsOp::getEffects( |
| 222 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 223 | consumesHandle(handles: getTargetMutable(), effects); |
| 224 | modifiesPayload(effects); |
| 225 | } |
| 226 | |
| 227 | //===----------------------------------------------------------------------===// |
| 228 | // Transform op registration |
| 229 | //===----------------------------------------------------------------------===// |
| 230 | |
| 231 | namespace { |
| 232 | class AffineTransformDialectExtension |
| 233 | : public transform::TransformDialectExtension< |
| 234 | AffineTransformDialectExtension> { |
| 235 | public: |
| 236 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AffineTransformDialectExtension) |
| 237 | |
| 238 | using Base::Base; |
| 239 | |
| 240 | void init() { |
| 241 | declareGeneratedDialect<AffineDialect>(); |
| 242 | declareGeneratedDialect<vector::VectorDialect>(); |
| 243 | |
| 244 | registerTransformOps< |
| 245 | #define GET_OP_LIST |
| 246 | #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.cpp.inc" |
| 247 | >(); |
| 248 | } |
| 249 | }; |
| 250 | } // namespace |
| 251 | |
| 252 | #define GET_OP_CLASSES |
| 253 | #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.cpp.inc" |
| 254 | |
| 255 | void mlir::affine::registerTransformDialectExtension( |
| 256 | DialectRegistry ®istry) { |
| 257 | registry.addExtensions<AffineTransformDialectExtension>(); |
| 258 | } |
| 259 | |