| 1 | //===- PaddingTilingInterface.cpp - Padding of TilingInterface ops --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Linalg/Transforms/Transforms.h" |
| 10 | |
| 11 | #include "mlir/Analysis/SliceAnalysis.h" |
| 12 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 13 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 14 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 15 | #include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" |
| 16 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 17 | #include "mlir/Dialect/UB/IR/UBMatchers.h" |
| 18 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 19 | #include "mlir/IR/AffineExpr.h" |
| 20 | #include "mlir/IR/BuiltinAttributes.h" |
| 21 | #include "mlir/IR/BuiltinTypeInterfaces.h" |
| 22 | #include "mlir/IR/BuiltinTypes.h" |
| 23 | #include "mlir/IR/OpDefinition.h" |
| 24 | #include "mlir/IR/Value.h" |
| 25 | #include "mlir/Interfaces/TilingInterface.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/Support/Casting.h" |
| 28 | |
| 29 | #define DEBUG_TYPE "pad-tiling-interface" |
| 30 | |
| 31 | using namespace mlir; |
| 32 | using namespace mlir::linalg; |
| 33 | using namespace mlir::tensor; |
| 34 | |
| 35 | #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ") |
| 36 | #define DBGSNL() (llvm::dbgs() << "\n") |
| 37 | |
| 38 | /// Form a "full-rank" padding specification so that the application is easy. |
| 39 | static SmallVector<OpFoldResult> |
| 40 | getFullRankPaddingSizes(Builder &b, ArrayRef<OpFoldResult> indexingSizes, |
| 41 | const PadTilingInterfaceOptions &options) { |
| 42 | SmallVector<OpFoldResult> paddingSizes; |
| 43 | // Complete the padding specification to specify all dimensions. |
| 44 | for (size_t idx = 0, e = indexingSizes.size(); idx != e; ++idx) { |
| 45 | // Complete to zero if needed. |
| 46 | paddingSizes.push_back(Elt: options.paddingSizes.size() > idx |
| 47 | ? options.paddingSizes[idx] |
| 48 | : b.getIndexAttr(value: 0)); |
| 49 | // If a dimension is zero (either specified or completed), replace by: |
| 50 | // - 1 if we are padding to the next multiple of. |
| 51 | // - indexingSizes[idx] otherwise |
| 52 | if (isZeroInteger(v: paddingSizes[idx])) { |
| 53 | paddingSizes[idx] = |
| 54 | options.padToMultipleOf ? b.getIndexAttr(value: 1) : indexingSizes[idx]; |
| 55 | } |
| 56 | LLVM_DEBUG(DBGS() << "----idx: " << idx << " : " << paddingSizes[idx] |
| 57 | << "\n" ); |
| 58 | } |
| 59 | return paddingSizes; |
| 60 | } |
| 61 | |
| 62 | /// Extracts the constant multiplier from an affine expression of the form |
| 63 | /// `d * c` or `c * d`, where `d` is an AffineDimExpr and `c` is an |
| 64 | /// AffineConstantExpr. Returns 1 if the expression is not a simple |
| 65 | /// multiplication of a dimension and a constant. |
| 66 | static int64_t (AffineExpr expr) { |
| 67 | if (auto binOp = dyn_cast<AffineBinaryOpExpr>(Val&: expr)) { |
| 68 | if (binOp.getKind() == AffineExprKind::Mul) { |
| 69 | auto lhsD = dyn_cast<AffineDimExpr>(Val: binOp.getLHS()); |
| 70 | auto rhsC = dyn_cast<AffineConstantExpr>(Val: binOp.getRHS()); |
| 71 | if (lhsD && rhsC) { |
| 72 | return rhsC.getValue(); |
| 73 | } |
| 74 | auto lhsC = dyn_cast<AffineConstantExpr>(Val: binOp.getLHS()); |
| 75 | auto rhsD = dyn_cast<AffineDimExpr>(Val: binOp.getRHS()); |
| 76 | if (lhsC && rhsD) { |
| 77 | return lhsC.getValue(); |
| 78 | } |
| 79 | } |
| 80 | } |
| 81 | return 1; |
| 82 | } |
| 83 | |
| 84 | /// Compute the padded shape of the given value `v` of `RankedTensorType` given |
| 85 | /// - `indexingSizes` a list of OpFoldResult. |
| 86 | /// - an `indexingMap` that encodes how the shape of varies with increases |
| 87 | /// in `indexingSizes`. |
| 88 | /// The `indexingMap` encodes how the shape of varies with `indexingSizes`. |
| 89 | /// The `indexingMap` + `indexingSizes` encoding suits StructuredOps. |
| 90 | /// The implementaiton below iteratively combines increases from contributing |
| 91 | /// dimensions using affine.apply operations. |
| 92 | /// The padded shape is computed by evaluating the maximum accessed index per |
| 93 | /// dimension, which may involve multiplying by constant factors derived from |
| 94 | /// the affine indexing expressions. Currently, only a limited set of projected |
| 95 | /// permutation indexing maps are supported, such as |
| 96 | /// - affine_map<(d0, d1, d2) -> (d0, d1)> |
| 97 | /// - affine_map<(d0, d1, d2) -> (d0, d1 + d2)> |
| 98 | /// - affine_map<(d0, d1) -> (d0 * 3 + d1)> |
| 99 | /// In the future, more general interfaces can be devised to encode similar |
| 100 | /// shape evolutions and map between an op and its operands. |
| 101 | SmallVector<OpFoldResult> |
| 102 | linalg::computePaddedShape(OpBuilder &builder, TypedValue<RankedTensorType> v, |
| 103 | AffineMap indexingMap, |
| 104 | ArrayRef<OpFoldResult> indexingSizes, |
| 105 | const PadTilingInterfaceOptions &options) { |
| 106 | Location loc = v.getLoc(); |
| 107 | SmallVector<OpFoldResult> paddedShape; |
| 108 | auto tensorType = cast<RankedTensorType>(Val: v.getType()); |
| 109 | paddedShape.resize_for_overwrite(N: tensorType.getRank()); |
| 110 | assert(tensorType.getRank() == indexingMap.getNumResults() && |
| 111 | "expect the number of results of the affine map to match the tensor " |
| 112 | "rank" ); |
| 113 | |
| 114 | // "Full-rank" padding specification. |
| 115 | SmallVector<OpFoldResult> paddingSizes = |
| 116 | getFullRankPaddingSizes(b&: builder, indexingSizes, options); |
| 117 | |
| 118 | // For each dimension in the operand's shape, iterate over indexingSizes and |
| 119 | // add the various term contributions. |
| 120 | for (const auto &enResults : enumerate(First: indexingMap.getResults())) { |
| 121 | int64_t resultIndex = enResults.index(); |
| 122 | AffineMap partialIndexingMap = indexingMap.getSubMap( |
| 123 | resultPos: ArrayRef<unsigned>{static_cast<unsigned>(resultIndex)}); |
| 124 | |
| 125 | LLVM_DEBUG(DBGS() << "----resultIndex: " << resultIndex |
| 126 | << " with partialIndexingMap: " << partialIndexingMap |
| 127 | << "\n" ); |
| 128 | |
| 129 | // Find all padding dimensions that contribute to this operand dimension |
| 130 | // and compute the padded term contribution to the final padded shape. |
| 131 | SmallVector<OpFoldResult> terms; |
| 132 | for (size_t paddingDim = 0, e = paddingSizes.size(); paddingDim != e; |
| 133 | ++paddingDim) { |
| 134 | OpFoldResult paddingSize = paddingSizes[paddingDim]; |
| 135 | LLVM_DEBUG(DBGS() << "------try apply padding of dim: " << paddingDim |
| 136 | << " to: " << paddingSize << "\n" ); |
| 137 | if (!enResults.value().isFunctionOfDim(position: paddingDim)) |
| 138 | continue; |
| 139 | |
| 140 | LLVM_DEBUG(DBGS() << "------apply padding of dim: " << paddingDim |
| 141 | << " to: " << paddingSize << "\n" ); |
| 142 | |
| 143 | // Project non-'paddingDim' dimensions and compress the result. |
| 144 | llvm::SmallBitVector projectedDims(partialIndexingMap.getNumDims(), true); |
| 145 | projectedDims.flip(Idx: paddingDim); |
| 146 | AffineMap projectedMap = |
| 147 | mlir::projectDims(map: partialIndexingMap, projectedDimensions: projectedDims, |
| 148 | /*compressDimsFlag=*/true); |
| 149 | |
| 150 | // If we are padding to the next multiple of, compose with ceil(sz) * sz. |
| 151 | OpFoldResult paddingDimOfr; |
| 152 | if (options.padToMultipleOf) { |
| 153 | AffineExpr d0, s0; |
| 154 | bindDims(ctx: builder.getContext(), exprs&: d0); |
| 155 | bindSymbols(ctx: builder.getContext(), exprs&: s0); |
| 156 | AffineMap ceilMap = AffineMap::get(dimCount: 1, symbolCount: 1, result: d0.ceilDiv(other: s0) * s0); |
| 157 | AffineMap composedMap = projectedMap.compose(map: ceilMap); |
| 158 | paddingDimOfr = affine::makeComposedFoldedAffineApply( |
| 159 | b&: builder, loc, map: composedMap, operands: {indexingSizes[paddingDim], paddingSize}, |
| 160 | /*composeAffineMin=*/true); |
| 161 | } else { |
| 162 | // Otherwise just set to paddingSize. |
| 163 | paddingDimOfr = affine::makeComposedFoldedAffineApply( |
| 164 | b&: builder, loc, map: projectedMap, operands: paddingSize); |
| 165 | } |
| 166 | |
| 167 | // Adjust for the maximum accessed index, which is (paddingSize - 1) * |
| 168 | // multiplier. |
| 169 | AffineExpr d0; |
| 170 | bindDims(ctx: builder.getContext(), exprs&: d0); |
| 171 | int64_t multiplier = extractConstantMultiplier(expr: projectedMap.getResult(idx: 0)); |
| 172 | AffineMap subtractMap = AffineMap::get(dimCount: 1, symbolCount: 0, result: d0 - multiplier); |
| 173 | OpFoldResult maxAccessIdx = affine::makeComposedFoldedAffineApply( |
| 174 | b&: builder, loc, map: subtractMap, operands: {paddingDimOfr}); |
| 175 | terms.push_back(Elt: maxAccessIdx); |
| 176 | |
| 177 | LLVM_DEBUG(DBGS() << "------new term: " << terms.back() << "\n" ); |
| 178 | } |
| 179 | |
| 180 | // If there are no terms, just return the dim. |
| 181 | if (terms.empty()) { |
| 182 | paddedShape[resultIndex] = |
| 183 | createFoldedDimOp(b&: builder, loc, val: v, dim: resultIndex); |
| 184 | continue; |
| 185 | } |
| 186 | |
| 187 | // Sum individual terms' contributions. |
| 188 | SmallVector<AffineExpr> dims(terms.size()); |
| 189 | bindDimsList(ctx: builder.getContext(), exprs: MutableArrayRef{dims}); |
| 190 | AffineExpr sumExpr = dims.front(); |
| 191 | for (unsigned i = 1; i < dims.size(); ++i) |
| 192 | sumExpr = sumExpr + dims[i]; |
| 193 | // Add 1 to the maximum accessed index and get the final padded size. |
| 194 | OpFoldResult paddedDimOfr = |
| 195 | affine::makeComposedFoldedAffineApply(b&: builder, loc, expr: sumExpr + 1, operands: terms); |
| 196 | paddedShape[resultIndex] = paddedDimOfr; |
| 197 | } |
| 198 | |
| 199 | return paddedShape; |
| 200 | } |
| 201 | |
| 202 | FailureOr<SmallVector<OpFoldResult>> |
| 203 | linalg::computeIndexingMapOpInterfacePaddedShape( |
| 204 | OpBuilder &builder, OpOperand &operandToPad, |
| 205 | ArrayRef<Range> iterationDomain, const PadTilingInterfaceOptions &options) { |
| 206 | auto transferOp = |
| 207 | llvm::dyn_cast<IndexingMapOpInterface>(Val: operandToPad.getOwner()); |
| 208 | if (!transferOp) |
| 209 | return failure(); |
| 210 | |
| 211 | // clang-format off |
| 212 | assert(llvm::all_of(iterationDomain, [&builder](Range r) { |
| 213 | return r.offset == OpFoldResult(builder.getIndexAttr(0)) && |
| 214 | r.stride == OpFoldResult(builder.getIndexAttr(1)); |
| 215 | }) && "expected 0-offset 1-stride loop ranges" ); |
| 216 | // clang-format on |
| 217 | SmallVector<OpFoldResult> loopUpperBounds; |
| 218 | loopUpperBounds.reserve(N: iterationDomain.size()); |
| 219 | for (const Range &range : iterationDomain) |
| 220 | loopUpperBounds.push_back(Elt: range.size); |
| 221 | |
| 222 | AffineMap indexingMap = transferOp.getMatchingIndexingMap(opOperand: &operandToPad); |
| 223 | return computePaddedShape( |
| 224 | builder, v: cast<TypedValue<RankedTensorType>>(Val: operandToPad.get()), |
| 225 | indexingMap, indexingSizes: loopUpperBounds, options); |
| 226 | } |
| 227 | |
| 228 | /// Pad a single operand to `paddedShape` using `paddingValueAttr` as padding |
| 229 | /// Value. |
| 230 | static Value padOperand(OpBuilder &builder, TilingInterface opToPad, |
| 231 | TypedValue<RankedTensorType> v, |
| 232 | ArrayRef<OpFoldResult> paddedShape, |
| 233 | Attribute paddingValueAttr) { |
| 234 | Value paddingValue; |
| 235 | if (auto complexTy = |
| 236 | dyn_cast<ComplexType>(Val: getElementTypeOrSelf(type: v.getType()))) { |
| 237 | if (auto complexAttr = dyn_cast<ArrayAttr>(Val&: paddingValueAttr)) { |
| 238 | paddingValue = complex::ConstantOp::create(builder, location: opToPad.getLoc(), |
| 239 | complex: complexTy, value: complexAttr); |
| 240 | } |
| 241 | } else if (matchPattern(attr: paddingValueAttr, pattern: ub::m_Poison())) { |
| 242 | paddingValue = ub::PoisonOp::create(builder, location: opToPad.getLoc(), |
| 243 | result: getElementTypeOrSelf(type: v.getType())); |
| 244 | } else if (auto typedAttr = dyn_cast<TypedAttr>(Val&: paddingValueAttr)) { |
| 245 | paddingValue = |
| 246 | arith::ConstantOp::create(builder, location: opToPad.getLoc(), value: typedAttr); |
| 247 | } |
| 248 | assert(paddingValue && "failed to create value from padding attribute" ); |
| 249 | |
| 250 | // Pad the operand to the bounding box defined by `paddedShape`. |
| 251 | SmallVector<int64_t> tensorShape; |
| 252 | SmallVector<Value> dynDims; |
| 253 | for (OpFoldResult ofr : paddedShape) { |
| 254 | std::optional<int64_t> cst = getConstantIntValue(ofr); |
| 255 | tensorShape.push_back(Elt: cst.has_value() ? *cst : ShapedType::kDynamic); |
| 256 | if (!cst.has_value()) |
| 257 | dynDims.push_back(Elt: dyn_cast<Value>(Val&: ofr)); |
| 258 | } |
| 259 | // TODO: use dispatchIndexOpFoldResults(paddedShape, dynDims, paddedShape); |
| 260 | |
| 261 | auto paddedTensorType = |
| 262 | RankedTensorType::get(shape: tensorShape, elementType: getElementTypeOrSelf(val: v)); |
| 263 | LLVM_DEBUG(DBGS() << "--SUCCESS, makeComposedPadHighOp with type: " |
| 264 | << paddedTensorType); |
| 265 | return makeComposedPadHighOp(b&: builder, loc: opToPad.getLoc(), type: paddedTensorType, source: v, |
| 266 | padding: paddingValue, /*nofold=*/false, typeDynDims: dynDims); |
| 267 | } |
| 268 | |
| 269 | /// Returns true if `operand` is indexed along at least one reduction dimension |
| 270 | /// of `linalgOp` |
| 271 | static bool isReducedOperand(linalg::LinalgOp linalgOp, OpOperand *operand, |
| 272 | ArrayRef<utils::IteratorType> iterTypes) { |
| 273 | assert(operand->getOwner() == linalgOp.getOperation() && |
| 274 | "expected operand to belong to the LinalgOp" ); |
| 275 | |
| 276 | AffineMap map = linalgOp.getMatchingIndexingMap(opOperand: operand); |
| 277 | return llvm::any_of(Range: llvm::enumerate(First&: iterTypes), P: [&](auto it) { |
| 278 | return it.value() == utils::IteratorType::reduction && |
| 279 | map.isFunctionOfDim(position: it.index()); |
| 280 | }); |
| 281 | } |
| 282 | |
| 283 | /// On success, returns a vector with pad values for every operand of linalgOp. |
| 284 | /// Fails if the body of `linalgOp` is not contraction-like, or if its |
| 285 | /// `elemwise`/`reduce` pair admits no padding value. |
| 286 | static FailureOr<SmallVector<Attribute>> |
| 287 | inferContractionPaddingValues(OpBuilder &builder, linalg::LinalgOp linalgOp, |
| 288 | ArrayRef<utils::IteratorType> iterTypes, |
| 289 | ArrayRef<Attribute> defaultPadVals) { |
| 290 | // NOTE: For contraction-like Ops, we infer the padding value by looking at |
| 291 | // both elemwise and reduce Ops, where (see isContractionBody for details): |
| 292 | // %0 = <elemwise>(permutation-of(cu(block-argument-0), |
| 293 | // cu(block-argument-1))) |
| 294 | // %1 = <reduce>(permutation-of(cu(%0), cu(block-argument-2))) |
| 295 | // return-like cu(%1) |
| 296 | Operation *elemwise = nullptr, *reduce = nullptr; |
| 297 | auto captureBodyOps = [&](Operation *e, Operation *r) { |
| 298 | elemwise = e; |
| 299 | reduce = r; |
| 300 | return true; |
| 301 | }; |
| 302 | if (!linalg::detail::isContractionBody(block&: *linalgOp.getBlock(), isaPair: captureBodyOps)) |
| 303 | return failure(); |
| 304 | |
| 305 | // Identify the padding value, `p`, for which `elemwise(p, x)` is the neutral |
| 306 | // element of `reduce` for every `x`. |
| 307 | // Not every contraction can be zero-padded: a body with `elemwise` = |
| 308 | // `arith.addf` and `reduce` = `arith.maximumf` (a max-plus contraction) pads |
| 309 | // with `-inf` instead. |
| 310 | auto getPadValue = [&](Type elementType) -> Attribute { |
| 311 | // `0 * x = 0` and `0 + acc = acc`. |
| 312 | if ((isa<arith::MulFOp>(Val: elemwise) && isa<arith::AddFOp>(Val: reduce)) || |
| 313 | (isa<arith::MulIOp>(Val: elemwise) && isa<arith::AddIOp>(Val: reduce)) || |
| 314 | (isa<complex::MulOp>(Val: elemwise) && isa<complex::AddOp>(Val: reduce))) |
| 315 | return builder.getZeroAttr(type: elementType); |
| 316 | // `false & x = false` and `false | acc = acc`. |
| 317 | if (isa<arith::AndIOp>(Val: elemwise) && isa<arith::OrIOp>(Val: reduce) && |
| 318 | elemwise->getResult(idx: 0).getType().isInteger(width: 1)) |
| 319 | return builder.getZeroAttr(type: elementType); |
| 320 | return {}; |
| 321 | }; |
| 322 | |
| 323 | SmallVector<Attribute> paddingValues(defaultPadVals.begin(), |
| 324 | defaultPadVals.end()); |
| 325 | for (OpOperand *input : linalgOp.getDpsInputOperands()) { |
| 326 | if (!isReducedOperand(linalgOp, operand: input, iterTypes)) |
| 327 | continue; |
| 328 | Attribute padValue = |
| 329 | getPadValue(getElementTypeOrSelf(type: input->get().getType())); |
| 330 | if (!padValue) |
| 331 | return failure(); |
| 332 | paddingValues[input->getOperandNumber()] = padValue; |
| 333 | } |
| 334 | return paddingValues; |
| 335 | } |
| 336 | |
| 337 | /// Infers a semantics-preserving padding value for every operand of `toPad` |
| 338 | /// (indexed by operand number). Operands that are reduced are padded with the |
| 339 | /// neutral element of their reduction combiner (e.g. `-inf` for `maximumf`, `1` |
| 340 | /// for `mulf`); every other operand is padded with the zero value of its |
| 341 | /// element type. |
| 342 | /// |
| 343 | /// Inference is conservative: it returns failure when a semantics-preserving |
| 344 | /// value cannot be determined (a non-LinalgOp reduction, or a reduction whose |
| 345 | /// neutral element is unknown), letting callers set `options.paddingValues` |
| 346 | /// explicitly instead. |
| 347 | static FailureOr<SmallVector<Attribute>> |
| 348 | inferPaddingValues(OpBuilder &builder, TilingInterface toPad) { |
| 349 | Operation *op = toPad.getOperation(); |
| 350 | |
| 351 | // Padding acts on operands; default each to the zero of its element type. |
| 352 | SmallVector<Attribute> paddingValues; |
| 353 | for (Type t : op->getOperandTypes()) |
| 354 | paddingValues.push_back(Elt: builder.getZeroAttr(type: getElementTypeOrSelf(type: t))); |
| 355 | |
| 356 | // No reduction: padded elements are never combined, so zero is always safe. |
| 357 | SmallVector<utils::IteratorType> iterTypes = toPad.getLoopIteratorTypes(); |
| 358 | if (!llvm::is_contained(Range&: iterTypes, Element: utils::IteratorType::reduction)) |
| 359 | return paddingValues; |
| 360 | |
| 361 | // A reduction's neutral element requires inspecting the combiner, which is |
| 362 | // only possible for a LinalgOp; fail conservatively otherwise. |
| 363 | auto linalgOp = dyn_cast<linalg::LinalgOp>(Val: op); |
| 364 | if (!linalgOp) |
| 365 | return failure(); |
| 366 | |
| 367 | // A contraction-like body accumulates as `acc = reduce(elemwise(a, b), acc)` |
| 368 | // and pads with a value derived from that pair. |
| 369 | FailureOr<SmallVector<Attribute>> contractionValues = |
| 370 | inferContractionPaddingValues(builder, linalgOp, iterTypes, |
| 371 | defaultPadVals: paddingValues); |
| 372 | if (succeeded(Result: contractionValues)) |
| 373 | return *contractionValues; |
| 374 | |
| 375 | // Only a single reduction has an unambiguous per-operand neutral. |
| 376 | if (linalgOp.getNumDpsInits() != 1) |
| 377 | return failure(); |
| 378 | |
| 379 | // For reduction-like bodies, the neutral element of the combiner is used to |
| 380 | // pad the reduced operands. |
| 381 | SmallVector<Operation *> combiners; |
| 382 | if (!matchReduction(iterCarriedArgs: linalgOp.getRegionOutputArgs(), /*redPos=*/0, combinerOps&: combiners)) |
| 383 | return failure(); |
| 384 | if (combiners.size() != 1) |
| 385 | return failure(); |
| 386 | Operation *combiner = combiners.front(); |
| 387 | |
| 388 | std::optional<TypedAttr> neutral = arith::getNeutralElement(op: combiner); |
| 389 | if (!neutral) |
| 390 | return failure(); |
| 391 | |
| 392 | // In case fastMath is enabled with `nnan`, the neutral element of |
| 393 | // `maxnumf`/`minnumf`, NaN, becomes an invalid padding value. Bail out and |
| 394 | // let the caller supply a value explicitly. |
| 395 | if (auto floatNeutral = dyn_cast<FloatAttr>(Val&: *neutral); |
| 396 | floatNeutral && floatNeutral.getValue().isNaN()) { |
| 397 | auto fastMath = dyn_cast<arith::ArithFastMathInterface>(Val: combiner); |
| 398 | if (fastMath && |
| 399 | arith::bitEnumContainsAny(bits: fastMath.getFastMathFlagsAttr().getValue(), |
| 400 | bit: arith::FastMathFlags::nnan)) |
| 401 | return failure(); |
| 402 | } |
| 403 | |
| 404 | for (OpOperand *input : linalgOp.getDpsInputOperands()) { |
| 405 | if (!isReducedOperand(linalgOp, operand: input, iterTypes)) |
| 406 | continue; |
| 407 | // A reduced operand must feed the combiner directly to use its neutral; an |
| 408 | // indirect one (e.g. via a math.exp) has no valid pad value -> fail. |
| 409 | if (!llvm::is_contained(Range: linalgOp.getMatchingBlockArgument(opOperand: input).getUsers(), |
| 410 | Element: combiner)) |
| 411 | return failure(); |
| 412 | paddingValues[input->getOperandNumber()] = *neutral; |
| 413 | } |
| 414 | return paddingValues; |
| 415 | } |
| 416 | |
| 417 | FailureOr<PadTilingInterfaceResult> linalg::rewriteAsPaddedOp( |
| 418 | OpBuilder &builder, TilingInterface toPad, |
| 419 | PadTilingInterfaceOptions options, |
| 420 | const PadSizeComputationFunction &computePaddingSizeFun) { |
| 421 | LLVM_DEBUG(DBGS() << "Start rewriteAsPaddedOp : " << toPad << "\n" ); |
| 422 | SmallVector<tensor::PadOp> padOps; |
| 423 | Location loc = toPad.getLoc(); |
| 424 | |
| 425 | // Allow inference of pad values if they are not explicitly specified. |
| 426 | if (options.paddingValues.empty()) { |
| 427 | FailureOr<SmallVector<Attribute>> inferred = |
| 428 | inferPaddingValues(builder, toPad); |
| 429 | if (failed(Result: inferred)) { |
| 430 | LLVM_DEBUG(DBGS() << "Could not infer pad values: FAIL\n" ); |
| 431 | return failure(); |
| 432 | } |
| 433 | options.paddingValues = std::move(*inferred); |
| 434 | } |
| 435 | |
| 436 | if (llvm::any_of(Range: toPad->getOperands(), |
| 437 | P: [](Value v) { return isa<MemRefType>(Val: v.getType()); })) { |
| 438 | LLVM_DEBUG(DBGS() << "Not an operation on tensors: FAIL\n" ); |
| 439 | return failure(); |
| 440 | } |
| 441 | |
| 442 | // 1. Get the loopUpperBounds from the TilingInterface. |
| 443 | SmallVector<Range> iterationDomain = toPad.getIterationDomain(b&: builder); |
| 444 | |
| 445 | // 2. For each operand. |
| 446 | SmallVector<Value> newOperands; |
| 447 | newOperands.reserve(N: toPad->getNumOperands()); |
| 448 | for (OpOperand &opOperand : toPad->getOpOperands()) { |
| 449 | Value operand = opOperand.get(); |
| 450 | LLVM_DEBUG(DBGS() << "--start padding operand: " << operand << "\n" ); |
| 451 | |
| 452 | // 2.a. Skip scalar-like operands. |
| 453 | Type operandType = operand.getType(); |
| 454 | if (!isa<RankedTensorType>(Val: operandType)) { |
| 455 | assert((!isa<ShapedType>(operandType) || isa<VectorType>(operandType)) && |
| 456 | "Unexpected non-vector ShapedType" ); |
| 457 | newOperands.push_back(Elt: operand); |
| 458 | continue; |
| 459 | } |
| 460 | |
| 461 | // 2.a. Compute padded shape. |
| 462 | FailureOr<SmallVector<OpFoldResult>> maybePaddedShape = |
| 463 | computePaddingSizeFun(builder, opOperand, iterationDomain, options); |
| 464 | if (failed(Result: maybePaddedShape)) { |
| 465 | LLVM_DEBUG(DBGS() << "Could not get padded shape of operand: FAIL\n" ); |
| 466 | return failure(); |
| 467 | } |
| 468 | |
| 469 | // 2.b. Expect proper `paddingValues`. |
| 470 | // TODO: we may want to allow garbage padding in the future, in which case |
| 471 | // we would just not assert. |
| 472 | if (opOperand.getOperandNumber() >= options.paddingValues.size()) { |
| 473 | LLVM_DEBUG(DBGS() << "Too few padding values specified: FAIL\n" ); |
| 474 | return failure(); |
| 475 | } |
| 476 | Attribute paddingValueAttr = |
| 477 | options.paddingValues[opOperand.getOperandNumber()]; |
| 478 | |
| 479 | // 2.c. Perform actual padding. |
| 480 | Value paddedOperand = |
| 481 | padOperand(builder, opToPad: toPad, v: cast<TypedValue<RankedTensorType>>(Val&: operand), |
| 482 | paddedShape: *maybePaddedShape, paddingValueAttr); |
| 483 | LLVM_DEBUG(DBGS() << "--done padding operand: " << paddedOperand << "\n" ); |
| 484 | |
| 485 | newOperands.push_back(Elt: paddedOperand); |
| 486 | if (auto padOp = paddedOperand.getDefiningOp<tensor::PadOp>()) |
| 487 | padOps.push_back(Elt: padOp); |
| 488 | } |
| 489 | |
| 490 | // 3. Form the resulting tensor::ExtractSliceOp. |
| 491 | ReifiedRankedShapedTypeDims reifiedResultShapes; |
| 492 | if (failed(Result: reifyResultShapes(b&: builder, op: toPad, reifiedReturnShapes&: reifiedResultShapes))) { |
| 493 | LLVM_DEBUG(DBGS() << "Failed to reify result shapes: FAIL\n" ); |
| 494 | return failure(); |
| 495 | } |
| 496 | assert(reifiedResultShapes.size() == toPad->getNumResults() && |
| 497 | "expected same number of results" ); |
| 498 | |
| 499 | // Clone `toPad` to operate on the statically padded shapes. |
| 500 | auto resultTensorTypes = |
| 501 | ValueRange(newOperands).take_back(n: toPad->getNumResults()).getTypes(); |
| 502 | // clone **should** properly notify the builder. |
| 503 | TilingInterface paddedOp = |
| 504 | clone(b&: builder, op: toPad, newResultTypes: resultTensorTypes, newOperands); |
| 505 | LLVM_DEBUG(DBGS() << "--cloned padded op: " << paddedOp << "\n" ); |
| 506 | |
| 507 | // Recover the slice out of the new static results. |
| 508 | SmallVector<Value> paddedSubtensorResults; |
| 509 | paddedSubtensorResults.reserve(N: toPad->getNumResults()); |
| 510 | for (const auto &en : llvm::enumerate(First: paddedOp->getResults())) { |
| 511 | Value paddedResult = en.value(); |
| 512 | int64_t resultNumber = en.index(); |
| 513 | int64_t rank = cast<RankedTensorType>(Val: paddedResult.getType()).getRank(); |
| 514 | SmallVector<OpFoldResult> offsets(rank, builder.getIndexAttr(value: 0)); |
| 515 | SmallVector<OpFoldResult> strides(rank, builder.getIndexAttr(value: 1)); |
| 516 | paddedSubtensorResults.push_back(Elt: tensor::ExtractSliceOp::create( |
| 517 | builder, location: loc, source: paddedResult, offsets, sizes: reifiedResultShapes[resultNumber], |
| 518 | strides)); |
| 519 | } |
| 520 | |
| 521 | return PadTilingInterfaceResult{.padOps: padOps, .paddedOp: paddedOp, .replacements: paddedSubtensorResults}; |
| 522 | } |
| 523 | |