1//===- PaddingTilingInterface.cpp - Padding of TilingInterface ops --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
10
11#include "mlir/Analysis/SliceAnalysis.h"
12#include "mlir/Dialect/Affine/IR/AffineOps.h"
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Complex/IR/Complex.h"
15#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
16#include "mlir/Dialect/Tensor/IR/Tensor.h"
17#include "mlir/Dialect/UB/IR/UBMatchers.h"
18#include "mlir/Dialect/Utils/StaticValueUtils.h"
19#include "mlir/IR/AffineExpr.h"
20#include "mlir/IR/BuiltinAttributes.h"
21#include "mlir/IR/BuiltinTypeInterfaces.h"
22#include "mlir/IR/BuiltinTypes.h"
23#include "mlir/IR/OpDefinition.h"
24#include "mlir/IR/Value.h"
25#include "mlir/Interfaces/TilingInterface.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/Support/Casting.h"
28
29#define DEBUG_TYPE "pad-tiling-interface"
30
31using namespace mlir;
32using namespace mlir::linalg;
33using namespace mlir::tensor;
34
35#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
36#define DBGSNL() (llvm::dbgs() << "\n")
37
38/// Form a "full-rank" padding specification so that the application is easy.
39static SmallVector<OpFoldResult>
40getFullRankPaddingSizes(Builder &b, ArrayRef<OpFoldResult> indexingSizes,
41 const PadTilingInterfaceOptions &options) {
42 SmallVector<OpFoldResult> paddingSizes;
43 // Complete the padding specification to specify all dimensions.
44 for (size_t idx = 0, e = indexingSizes.size(); idx != e; ++idx) {
45 // Complete to zero if needed.
46 paddingSizes.push_back(Elt: options.paddingSizes.size() > idx
47 ? options.paddingSizes[idx]
48 : b.getIndexAttr(value: 0));
49 // If a dimension is zero (either specified or completed), replace by:
50 // - 1 if we are padding to the next multiple of.
51 // - indexingSizes[idx] otherwise
52 if (isZeroInteger(v: paddingSizes[idx])) {
53 paddingSizes[idx] =
54 options.padToMultipleOf ? b.getIndexAttr(value: 1) : indexingSizes[idx];
55 }
56 LLVM_DEBUG(DBGS() << "----idx: " << idx << " : " << paddingSizes[idx]
57 << "\n");
58 }
59 return paddingSizes;
60}
61
62/// Extracts the constant multiplier from an affine expression of the form
63/// `d * c` or `c * d`, where `d` is an AffineDimExpr and `c` is an
64/// AffineConstantExpr. Returns 1 if the expression is not a simple
65/// multiplication of a dimension and a constant.
66static int64_t extractConstantMultiplier(AffineExpr expr) {
67 if (auto binOp = dyn_cast<AffineBinaryOpExpr>(Val&: expr)) {
68 if (binOp.getKind() == AffineExprKind::Mul) {
69 auto lhsD = dyn_cast<AffineDimExpr>(Val: binOp.getLHS());
70 auto rhsC = dyn_cast<AffineConstantExpr>(Val: binOp.getRHS());
71 if (lhsD && rhsC) {
72 return rhsC.getValue();
73 }
74 auto lhsC = dyn_cast<AffineConstantExpr>(Val: binOp.getLHS());
75 auto rhsD = dyn_cast<AffineDimExpr>(Val: binOp.getRHS());
76 if (lhsC && rhsD) {
77 return lhsC.getValue();
78 }
79 }
80 }
81 return 1;
82}
83
84/// Compute the padded shape of the given value `v` of `RankedTensorType` given
85/// - `indexingSizes` a list of OpFoldResult.
86/// - an `indexingMap` that encodes how the shape of varies with increases
87/// in `indexingSizes`.
88/// The `indexingMap` encodes how the shape of varies with `indexingSizes`.
89/// The `indexingMap` + `indexingSizes` encoding suits StructuredOps.
90/// The implementaiton below iteratively combines increases from contributing
91/// dimensions using affine.apply operations.
92/// The padded shape is computed by evaluating the maximum accessed index per
93/// dimension, which may involve multiplying by constant factors derived from
94/// the affine indexing expressions. Currently, only a limited set of projected
95/// permutation indexing maps are supported, such as
96/// - affine_map<(d0, d1, d2) -> (d0, d1)>
97/// - affine_map<(d0, d1, d2) -> (d0, d1 + d2)>
98/// - affine_map<(d0, d1) -> (d0 * 3 + d1)>
99/// In the future, more general interfaces can be devised to encode similar
100/// shape evolutions and map between an op and its operands.
101SmallVector<OpFoldResult>
102linalg::computePaddedShape(OpBuilder &builder, TypedValue<RankedTensorType> v,
103 AffineMap indexingMap,
104 ArrayRef<OpFoldResult> indexingSizes,
105 const PadTilingInterfaceOptions &options) {
106 Location loc = v.getLoc();
107 SmallVector<OpFoldResult> paddedShape;
108 auto tensorType = cast<RankedTensorType>(Val: v.getType());
109 paddedShape.resize_for_overwrite(N: tensorType.getRank());
110 assert(tensorType.getRank() == indexingMap.getNumResults() &&
111 "expect the number of results of the affine map to match the tensor "
112 "rank");
113
114 // "Full-rank" padding specification.
115 SmallVector<OpFoldResult> paddingSizes =
116 getFullRankPaddingSizes(b&: builder, indexingSizes, options);
117
118 // For each dimension in the operand's shape, iterate over indexingSizes and
119 // add the various term contributions.
120 for (const auto &enResults : enumerate(First: indexingMap.getResults())) {
121 int64_t resultIndex = enResults.index();
122 AffineMap partialIndexingMap = indexingMap.getSubMap(
123 resultPos: ArrayRef<unsigned>{static_cast<unsigned>(resultIndex)});
124
125 LLVM_DEBUG(DBGS() << "----resultIndex: " << resultIndex
126 << " with partialIndexingMap: " << partialIndexingMap
127 << "\n");
128
129 // Find all padding dimensions that contribute to this operand dimension
130 // and compute the padded term contribution to the final padded shape.
131 SmallVector<OpFoldResult> terms;
132 for (size_t paddingDim = 0, e = paddingSizes.size(); paddingDim != e;
133 ++paddingDim) {
134 OpFoldResult paddingSize = paddingSizes[paddingDim];
135 LLVM_DEBUG(DBGS() << "------try apply padding of dim: " << paddingDim
136 << " to: " << paddingSize << "\n");
137 if (!enResults.value().isFunctionOfDim(position: paddingDim))
138 continue;
139
140 LLVM_DEBUG(DBGS() << "------apply padding of dim: " << paddingDim
141 << " to: " << paddingSize << "\n");
142
143 // Project non-'paddingDim' dimensions and compress the result.
144 llvm::SmallBitVector projectedDims(partialIndexingMap.getNumDims(), true);
145 projectedDims.flip(Idx: paddingDim);
146 AffineMap projectedMap =
147 mlir::projectDims(map: partialIndexingMap, projectedDimensions: projectedDims,
148 /*compressDimsFlag=*/true);
149
150 // If we are padding to the next multiple of, compose with ceil(sz) * sz.
151 OpFoldResult paddingDimOfr;
152 if (options.padToMultipleOf) {
153 AffineExpr d0, s0;
154 bindDims(ctx: builder.getContext(), exprs&: d0);
155 bindSymbols(ctx: builder.getContext(), exprs&: s0);
156 AffineMap ceilMap = AffineMap::get(dimCount: 1, symbolCount: 1, result: d0.ceilDiv(other: s0) * s0);
157 AffineMap composedMap = projectedMap.compose(map: ceilMap);
158 paddingDimOfr = affine::makeComposedFoldedAffineApply(
159 b&: builder, loc, map: composedMap, operands: {indexingSizes[paddingDim], paddingSize},
160 /*composeAffineMin=*/true);
161 } else {
162 // Otherwise just set to paddingSize.
163 paddingDimOfr = affine::makeComposedFoldedAffineApply(
164 b&: builder, loc, map: projectedMap, operands: paddingSize);
165 }
166
167 // Adjust for the maximum accessed index, which is (paddingSize - 1) *
168 // multiplier.
169 AffineExpr d0;
170 bindDims(ctx: builder.getContext(), exprs&: d0);
171 int64_t multiplier = extractConstantMultiplier(expr: projectedMap.getResult(idx: 0));
172 AffineMap subtractMap = AffineMap::get(dimCount: 1, symbolCount: 0, result: d0 - multiplier);
173 OpFoldResult maxAccessIdx = affine::makeComposedFoldedAffineApply(
174 b&: builder, loc, map: subtractMap, operands: {paddingDimOfr});
175 terms.push_back(Elt: maxAccessIdx);
176
177 LLVM_DEBUG(DBGS() << "------new term: " << terms.back() << "\n");
178 }
179
180 // If there are no terms, just return the dim.
181 if (terms.empty()) {
182 paddedShape[resultIndex] =
183 createFoldedDimOp(b&: builder, loc, val: v, dim: resultIndex);
184 continue;
185 }
186
187 // Sum individual terms' contributions.
188 SmallVector<AffineExpr> dims(terms.size());
189 bindDimsList(ctx: builder.getContext(), exprs: MutableArrayRef{dims});
190 AffineExpr sumExpr = dims.front();
191 for (unsigned i = 1; i < dims.size(); ++i)
192 sumExpr = sumExpr + dims[i];
193 // Add 1 to the maximum accessed index and get the final padded size.
194 OpFoldResult paddedDimOfr =
195 affine::makeComposedFoldedAffineApply(b&: builder, loc, expr: sumExpr + 1, operands: terms);
196 paddedShape[resultIndex] = paddedDimOfr;
197 }
198
199 return paddedShape;
200}
201
202FailureOr<SmallVector<OpFoldResult>>
203linalg::computeIndexingMapOpInterfacePaddedShape(
204 OpBuilder &builder, OpOperand &operandToPad,
205 ArrayRef<Range> iterationDomain, const PadTilingInterfaceOptions &options) {
206 auto transferOp =
207 llvm::dyn_cast<IndexingMapOpInterface>(Val: operandToPad.getOwner());
208 if (!transferOp)
209 return failure();
210
211 // clang-format off
212 assert(llvm::all_of(iterationDomain, [&builder](Range r) {
213 return r.offset == OpFoldResult(builder.getIndexAttr(0)) &&
214 r.stride == OpFoldResult(builder.getIndexAttr(1));
215 }) && "expected 0-offset 1-stride loop ranges");
216 // clang-format on
217 SmallVector<OpFoldResult> loopUpperBounds;
218 loopUpperBounds.reserve(N: iterationDomain.size());
219 for (const Range &range : iterationDomain)
220 loopUpperBounds.push_back(Elt: range.size);
221
222 AffineMap indexingMap = transferOp.getMatchingIndexingMap(opOperand: &operandToPad);
223 return computePaddedShape(
224 builder, v: cast<TypedValue<RankedTensorType>>(Val: operandToPad.get()),
225 indexingMap, indexingSizes: loopUpperBounds, options);
226}
227
228/// Pad a single operand to `paddedShape` using `paddingValueAttr` as padding
229/// Value.
230static Value padOperand(OpBuilder &builder, TilingInterface opToPad,
231 TypedValue<RankedTensorType> v,
232 ArrayRef<OpFoldResult> paddedShape,
233 Attribute paddingValueAttr) {
234 Value paddingValue;
235 if (auto complexTy =
236 dyn_cast<ComplexType>(Val: getElementTypeOrSelf(type: v.getType()))) {
237 if (auto complexAttr = dyn_cast<ArrayAttr>(Val&: paddingValueAttr)) {
238 paddingValue = complex::ConstantOp::create(builder, location: opToPad.getLoc(),
239 complex: complexTy, value: complexAttr);
240 }
241 } else if (matchPattern(attr: paddingValueAttr, pattern: ub::m_Poison())) {
242 paddingValue = ub::PoisonOp::create(builder, location: opToPad.getLoc(),
243 result: getElementTypeOrSelf(type: v.getType()));
244 } else if (auto typedAttr = dyn_cast<TypedAttr>(Val&: paddingValueAttr)) {
245 paddingValue =
246 arith::ConstantOp::create(builder, location: opToPad.getLoc(), value: typedAttr);
247 }
248 assert(paddingValue && "failed to create value from padding attribute");
249
250 // Pad the operand to the bounding box defined by `paddedShape`.
251 SmallVector<int64_t> tensorShape;
252 SmallVector<Value> dynDims;
253 for (OpFoldResult ofr : paddedShape) {
254 std::optional<int64_t> cst = getConstantIntValue(ofr);
255 tensorShape.push_back(Elt: cst.has_value() ? *cst : ShapedType::kDynamic);
256 if (!cst.has_value())
257 dynDims.push_back(Elt: dyn_cast<Value>(Val&: ofr));
258 }
259 // TODO: use dispatchIndexOpFoldResults(paddedShape, dynDims, paddedShape);
260
261 auto paddedTensorType =
262 RankedTensorType::get(shape: tensorShape, elementType: getElementTypeOrSelf(val: v));
263 LLVM_DEBUG(DBGS() << "--SUCCESS, makeComposedPadHighOp with type: "
264 << paddedTensorType);
265 return makeComposedPadHighOp(b&: builder, loc: opToPad.getLoc(), type: paddedTensorType, source: v,
266 padding: paddingValue, /*nofold=*/false, typeDynDims: dynDims);
267}
268
269/// Returns true if `operand` is indexed along at least one reduction dimension
270/// of `linalgOp`
271static bool isReducedOperand(linalg::LinalgOp linalgOp, OpOperand *operand,
272 ArrayRef<utils::IteratorType> iterTypes) {
273 assert(operand->getOwner() == linalgOp.getOperation() &&
274 "expected operand to belong to the LinalgOp");
275
276 AffineMap map = linalgOp.getMatchingIndexingMap(opOperand: operand);
277 return llvm::any_of(Range: llvm::enumerate(First&: iterTypes), P: [&](auto it) {
278 return it.value() == utils::IteratorType::reduction &&
279 map.isFunctionOfDim(position: it.index());
280 });
281}
282
283/// On success, returns a vector with pad values for every operand of linalgOp.
284/// Fails if the body of `linalgOp` is not contraction-like, or if its
285/// `elemwise`/`reduce` pair admits no padding value.
286static FailureOr<SmallVector<Attribute>>
287inferContractionPaddingValues(OpBuilder &builder, linalg::LinalgOp linalgOp,
288 ArrayRef<utils::IteratorType> iterTypes,
289 ArrayRef<Attribute> defaultPadVals) {
290 // NOTE: For contraction-like Ops, we infer the padding value by looking at
291 // both elemwise and reduce Ops, where (see isContractionBody for details):
292 // %0 = <elemwise>(permutation-of(cu(block-argument-0),
293 // cu(block-argument-1)))
294 // %1 = <reduce>(permutation-of(cu(%0), cu(block-argument-2)))
295 // return-like cu(%1)
296 Operation *elemwise = nullptr, *reduce = nullptr;
297 auto captureBodyOps = [&](Operation *e, Operation *r) {
298 elemwise = e;
299 reduce = r;
300 return true;
301 };
302 if (!linalg::detail::isContractionBody(block&: *linalgOp.getBlock(), isaPair: captureBodyOps))
303 return failure();
304
305 // Identify the padding value, `p`, for which `elemwise(p, x)` is the neutral
306 // element of `reduce` for every `x`.
307 // Not every contraction can be zero-padded: a body with `elemwise` =
308 // `arith.addf` and `reduce` = `arith.maximumf` (a max-plus contraction) pads
309 // with `-inf` instead.
310 auto getPadValue = [&](Type elementType) -> Attribute {
311 // `0 * x = 0` and `0 + acc = acc`.
312 if ((isa<arith::MulFOp>(Val: elemwise) && isa<arith::AddFOp>(Val: reduce)) ||
313 (isa<arith::MulIOp>(Val: elemwise) && isa<arith::AddIOp>(Val: reduce)) ||
314 (isa<complex::MulOp>(Val: elemwise) && isa<complex::AddOp>(Val: reduce)))
315 return builder.getZeroAttr(type: elementType);
316 // `false & x = false` and `false | acc = acc`.
317 if (isa<arith::AndIOp>(Val: elemwise) && isa<arith::OrIOp>(Val: reduce) &&
318 elemwise->getResult(idx: 0).getType().isInteger(width: 1))
319 return builder.getZeroAttr(type: elementType);
320 return {};
321 };
322
323 SmallVector<Attribute> paddingValues(defaultPadVals.begin(),
324 defaultPadVals.end());
325 for (OpOperand *input : linalgOp.getDpsInputOperands()) {
326 if (!isReducedOperand(linalgOp, operand: input, iterTypes))
327 continue;
328 Attribute padValue =
329 getPadValue(getElementTypeOrSelf(type: input->get().getType()));
330 if (!padValue)
331 return failure();
332 paddingValues[input->getOperandNumber()] = padValue;
333 }
334 return paddingValues;
335}
336
337/// Infers a semantics-preserving padding value for every operand of `toPad`
338/// (indexed by operand number). Operands that are reduced are padded with the
339/// neutral element of their reduction combiner (e.g. `-inf` for `maximumf`, `1`
340/// for `mulf`); every other operand is padded with the zero value of its
341/// element type.
342///
343/// Inference is conservative: it returns failure when a semantics-preserving
344/// value cannot be determined (a non-LinalgOp reduction, or a reduction whose
345/// neutral element is unknown), letting callers set `options.paddingValues`
346/// explicitly instead.
347static FailureOr<SmallVector<Attribute>>
348inferPaddingValues(OpBuilder &builder, TilingInterface toPad) {
349 Operation *op = toPad.getOperation();
350
351 // Padding acts on operands; default each to the zero of its element type.
352 SmallVector<Attribute> paddingValues;
353 for (Type t : op->getOperandTypes())
354 paddingValues.push_back(Elt: builder.getZeroAttr(type: getElementTypeOrSelf(type: t)));
355
356 // No reduction: padded elements are never combined, so zero is always safe.
357 SmallVector<utils::IteratorType> iterTypes = toPad.getLoopIteratorTypes();
358 if (!llvm::is_contained(Range&: iterTypes, Element: utils::IteratorType::reduction))
359 return paddingValues;
360
361 // A reduction's neutral element requires inspecting the combiner, which is
362 // only possible for a LinalgOp; fail conservatively otherwise.
363 auto linalgOp = dyn_cast<linalg::LinalgOp>(Val: op);
364 if (!linalgOp)
365 return failure();
366
367 // A contraction-like body accumulates as `acc = reduce(elemwise(a, b), acc)`
368 // and pads with a value derived from that pair.
369 FailureOr<SmallVector<Attribute>> contractionValues =
370 inferContractionPaddingValues(builder, linalgOp, iterTypes,
371 defaultPadVals: paddingValues);
372 if (succeeded(Result: contractionValues))
373 return *contractionValues;
374
375 // Only a single reduction has an unambiguous per-operand neutral.
376 if (linalgOp.getNumDpsInits() != 1)
377 return failure();
378
379 // For reduction-like bodies, the neutral element of the combiner is used to
380 // pad the reduced operands.
381 SmallVector<Operation *> combiners;
382 if (!matchReduction(iterCarriedArgs: linalgOp.getRegionOutputArgs(), /*redPos=*/0, combinerOps&: combiners))
383 return failure();
384 if (combiners.size() != 1)
385 return failure();
386 Operation *combiner = combiners.front();
387
388 std::optional<TypedAttr> neutral = arith::getNeutralElement(op: combiner);
389 if (!neutral)
390 return failure();
391
392 // In case fastMath is enabled with `nnan`, the neutral element of
393 // `maxnumf`/`minnumf`, NaN, becomes an invalid padding value. Bail out and
394 // let the caller supply a value explicitly.
395 if (auto floatNeutral = dyn_cast<FloatAttr>(Val&: *neutral);
396 floatNeutral && floatNeutral.getValue().isNaN()) {
397 auto fastMath = dyn_cast<arith::ArithFastMathInterface>(Val: combiner);
398 if (fastMath &&
399 arith::bitEnumContainsAny(bits: fastMath.getFastMathFlagsAttr().getValue(),
400 bit: arith::FastMathFlags::nnan))
401 return failure();
402 }
403
404 for (OpOperand *input : linalgOp.getDpsInputOperands()) {
405 if (!isReducedOperand(linalgOp, operand: input, iterTypes))
406 continue;
407 // A reduced operand must feed the combiner directly to use its neutral; an
408 // indirect one (e.g. via a math.exp) has no valid pad value -> fail.
409 if (!llvm::is_contained(Range: linalgOp.getMatchingBlockArgument(opOperand: input).getUsers(),
410 Element: combiner))
411 return failure();
412 paddingValues[input->getOperandNumber()] = *neutral;
413 }
414 return paddingValues;
415}
416
417FailureOr<PadTilingInterfaceResult> linalg::rewriteAsPaddedOp(
418 OpBuilder &builder, TilingInterface toPad,
419 PadTilingInterfaceOptions options,
420 const PadSizeComputationFunction &computePaddingSizeFun) {
421 LLVM_DEBUG(DBGS() << "Start rewriteAsPaddedOp : " << toPad << "\n");
422 SmallVector<tensor::PadOp> padOps;
423 Location loc = toPad.getLoc();
424
425 // Allow inference of pad values if they are not explicitly specified.
426 if (options.paddingValues.empty()) {
427 FailureOr<SmallVector<Attribute>> inferred =
428 inferPaddingValues(builder, toPad);
429 if (failed(Result: inferred)) {
430 LLVM_DEBUG(DBGS() << "Could not infer pad values: FAIL\n");
431 return failure();
432 }
433 options.paddingValues = std::move(*inferred);
434 }
435
436 if (llvm::any_of(Range: toPad->getOperands(),
437 P: [](Value v) { return isa<MemRefType>(Val: v.getType()); })) {
438 LLVM_DEBUG(DBGS() << "Not an operation on tensors: FAIL\n");
439 return failure();
440 }
441
442 // 1. Get the loopUpperBounds from the TilingInterface.
443 SmallVector<Range> iterationDomain = toPad.getIterationDomain(b&: builder);
444
445 // 2. For each operand.
446 SmallVector<Value> newOperands;
447 newOperands.reserve(N: toPad->getNumOperands());
448 for (OpOperand &opOperand : toPad->getOpOperands()) {
449 Value operand = opOperand.get();
450 LLVM_DEBUG(DBGS() << "--start padding operand: " << operand << "\n");
451
452 // 2.a. Skip scalar-like operands.
453 Type operandType = operand.getType();
454 if (!isa<RankedTensorType>(Val: operandType)) {
455 assert((!isa<ShapedType>(operandType) || isa<VectorType>(operandType)) &&
456 "Unexpected non-vector ShapedType");
457 newOperands.push_back(Elt: operand);
458 continue;
459 }
460
461 // 2.a. Compute padded shape.
462 FailureOr<SmallVector<OpFoldResult>> maybePaddedShape =
463 computePaddingSizeFun(builder, opOperand, iterationDomain, options);
464 if (failed(Result: maybePaddedShape)) {
465 LLVM_DEBUG(DBGS() << "Could not get padded shape of operand: FAIL\n");
466 return failure();
467 }
468
469 // 2.b. Expect proper `paddingValues`.
470 // TODO: we may want to allow garbage padding in the future, in which case
471 // we would just not assert.
472 if (opOperand.getOperandNumber() >= options.paddingValues.size()) {
473 LLVM_DEBUG(DBGS() << "Too few padding values specified: FAIL\n");
474 return failure();
475 }
476 Attribute paddingValueAttr =
477 options.paddingValues[opOperand.getOperandNumber()];
478
479 // 2.c. Perform actual padding.
480 Value paddedOperand =
481 padOperand(builder, opToPad: toPad, v: cast<TypedValue<RankedTensorType>>(Val&: operand),
482 paddedShape: *maybePaddedShape, paddingValueAttr);
483 LLVM_DEBUG(DBGS() << "--done padding operand: " << paddedOperand << "\n");
484
485 newOperands.push_back(Elt: paddedOperand);
486 if (auto padOp = paddedOperand.getDefiningOp<tensor::PadOp>())
487 padOps.push_back(Elt: padOp);
488 }
489
490 // 3. Form the resulting tensor::ExtractSliceOp.
491 ReifiedRankedShapedTypeDims reifiedResultShapes;
492 if (failed(Result: reifyResultShapes(b&: builder, op: toPad, reifiedReturnShapes&: reifiedResultShapes))) {
493 LLVM_DEBUG(DBGS() << "Failed to reify result shapes: FAIL\n");
494 return failure();
495 }
496 assert(reifiedResultShapes.size() == toPad->getNumResults() &&
497 "expected same number of results");
498
499 // Clone `toPad` to operate on the statically padded shapes.
500 auto resultTensorTypes =
501 ValueRange(newOperands).take_back(n: toPad->getNumResults()).getTypes();
502 // clone **should** properly notify the builder.
503 TilingInterface paddedOp =
504 clone(b&: builder, op: toPad, newResultTypes: resultTensorTypes, newOperands);
505 LLVM_DEBUG(DBGS() << "--cloned padded op: " << paddedOp << "\n");
506
507 // Recover the slice out of the new static results.
508 SmallVector<Value> paddedSubtensorResults;
509 paddedSubtensorResults.reserve(N: toPad->getNumResults());
510 for (const auto &en : llvm::enumerate(First: paddedOp->getResults())) {
511 Value paddedResult = en.value();
512 int64_t resultNumber = en.index();
513 int64_t rank = cast<RankedTensorType>(Val: paddedResult.getType()).getRank();
514 SmallVector<OpFoldResult> offsets(rank, builder.getIndexAttr(value: 0));
515 SmallVector<OpFoldResult> strides(rank, builder.getIndexAttr(value: 1));
516 paddedSubtensorResults.push_back(Elt: tensor::ExtractSliceOp::create(
517 builder, location: loc, source: paddedResult, offsets, sizes: reifiedResultShapes[resultNumber],
518 strides));
519 }
520
521 return PadTilingInterfaceResult{.padOps: padOps, .paddedOp: paddedOp, .replacements: paddedSubtensorResults};
522}
523