1//===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h"
10
11#include "mlir/Dialect/Affine/IR/AffineOps.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
14#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
15#include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h"
16#include "mlir/Dialect/Linalg/IR/Linalg.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/Tensor/IR/Tensor.h"
19#include "mlir/Dialect/Tensor/Transforms/SubsetOpInterfaceImpl.h"
20#include "mlir/Dialect/Utils/StaticValueUtils.h"
21#include "mlir/IR/BuiltinTypeInterfaces.h"
22#include "mlir/IR/Dialect.h"
23#include "mlir/IR/Operation.h"
24
25using namespace mlir;
26using namespace mlir::bufferization;
27using namespace mlir::tensor;
28
29namespace mlir {
30namespace tensor {
31namespace {
32
33struct CastOpInterface
34 : public BufferizableOpInterface::ExternalModel<CastOpInterface,
35 tensor::CastOp> {
36 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
37 const AnalysisState &state) const {
38 return false;
39 }
40
41 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
42 const AnalysisState &state) const {
43 return false;
44 }
45
46 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
47 const AnalysisState &state) const {
48 return {{op->getResult(idx: 0), BufferRelation::Equivalent}};
49 }
50
51 FailureOr<BufferLikeType>
52 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
53 const BufferizationState &state,
54 SmallVector<Value> &invocationStack) const {
55 auto castOp = cast<tensor::CastOp>(Val: op);
56 auto maybeSrcBufferType =
57 bufferization::detail::asMemRefType(bufferType: bufferization::getBufferType(
58 value: castOp.getSource(), options, state, invocationStack));
59 if (failed(Result: maybeSrcBufferType))
60 return failure();
61 Attribute memorySpace = maybeSrcBufferType->getMemorySpace();
62
63 // Note: `getMemRefTypeWithFullyDynamicLayout` returns an unranked memref
64 // type in case the input is an unranked tensor type.
65
66 // Case 1: Casting an unranked tensor
67 if (isa<UnrankedTensorType>(Val: castOp.getSource().getType())) {
68 // When casting to a ranked tensor, we cannot infer any static offset or
69 // strides from the source. Assume fully dynamic.
70 return cast<BufferLikeType>(
71 Val: getMemRefTypeWithFullyDynamicLayout(tensorType: castOp.getType(), memorySpace));
72 }
73
74 // Case 2: Casting to an unranked tensor type
75 if (isa<UnrankedTensorType>(Val: castOp.getType())) {
76 return cast<BufferLikeType>(
77 Val: getMemRefTypeWithFullyDynamicLayout(tensorType: castOp.getType(), memorySpace));
78 }
79
80 // Case 3: Ranked tensor -> ranked tensor. The offsets and strides do not
81 // change.
82 auto rankedResultType = cast<RankedTensorType>(Val: castOp.getType());
83 return cast<BufferLikeType>(Val: MemRefType::get(
84 shape: rankedResultType.getShape(), elementType: rankedResultType.getElementType(),
85 layout: llvm::cast<MemRefType>(Val&: *maybeSrcBufferType).getLayout(), memorySpace));
86 }
87
88 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
89 const BufferizationOptions &options,
90 BufferizationState &state) const {
91 auto castOp = cast<tensor::CastOp>(Val: op);
92
93 // The result buffer still has the old (pre-cast) type.
94 FailureOr<Value> resultBuffer =
95 getBuffer(rewriter, value: castOp.getSource(), options, state);
96 if (failed(Result: resultBuffer))
97 return failure();
98
99 // Compute the new type.
100 auto resultMemRefType =
101 bufferization::getBufferType(value: castOp.getResult(), options, state);
102 if (failed(Result: resultMemRefType))
103 return failure();
104 if (resultBuffer->getType() == *resultMemRefType) {
105 // This cast is a no-op.
106 replaceOpWithBufferizedValues(rewriter, op, values: *resultBuffer);
107 return success();
108 }
109
110 // Replace the op with a memref.cast.
111 assert(memref::CastOp::areCastCompatible(resultBuffer->getType(),
112 *resultMemRefType) &&
113 "CallOp::bufferize: cast incompatible");
114 replaceOpWithNewBufferizedOp<memref::CastOp>(
115 rewriter, op, args&: *resultMemRefType, args&: *resultBuffer);
116
117 return success();
118 }
119};
120
121/// Bufferization of tensor.collapse_shape. Replace with memref.collapse_shape.
122struct CollapseShapeOpInterface
123 : public BufferizableOpInterface::ExternalModel<CollapseShapeOpInterface,
124 tensor::CollapseShapeOp> {
125 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
126 const AnalysisState &state) const {
127 // tensor.collapse_shape may reallocate, at which point the source buffer is
128 // copied. I.e., there will be a memory read side effect on the bufferized
129 // source. This function conservatively returns "true" because whether a
130 // copy will be created or not is not known at this point.
131 return true;
132 }
133
134 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
135 const AnalysisState &state) const {
136 return false;
137 }
138
139 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
140 const AnalysisState &state) const {
141 // TODO: CollapseShapeOp may allocate at runtime.
142 return {{op->getOpResult(idx: 0), BufferRelation::Equivalent}};
143 }
144
145 FailureOr<BufferLikeType>
146 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
147 const BufferizationState &state,
148 SmallVector<Value> &invocationStack) const {
149 auto collapseShapeOp = cast<tensor::CollapseShapeOp>(Val: op);
150 auto maybeSrcBufferType = bufferization::getBufferType(
151 value: collapseShapeOp.getSrc(), options, state, invocationStack);
152 if (failed(Result: maybeSrcBufferType))
153 return failure();
154 auto srcBufferType = llvm::cast<MemRefType>(Val&: *maybeSrcBufferType);
155 bool canBeCollapsed = memref::CollapseShapeOp::isGuaranteedCollapsible(
156 srcType: srcBufferType, reassociation: collapseShapeOp.getReassociationIndices());
157
158 if (!canBeCollapsed) {
159 // If dims cannot be collapsed, this op bufferizes to a new allocation.
160 RankedTensorType tensorResultType = collapseShapeOp.getResultType();
161 return cast<BufferLikeType>(
162 Val: bufferization::getMemRefTypeWithStaticIdentityLayout(
163 tensorType: tensorResultType, memorySpace: srcBufferType.getMemorySpace()));
164 }
165
166 return cast<BufferLikeType>(Val: memref::CollapseShapeOp::computeCollapsedType(
167 srcType: srcBufferType, reassociation: collapseShapeOp.getReassociationIndices()));
168 }
169
170 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
171 const BufferizationOptions &options,
172 BufferizationState &state) const {
173 auto collapseShapeOp = cast<tensor::CollapseShapeOp>(Val: op);
174 RankedTensorType tensorResultType = collapseShapeOp.getResultType();
175 FailureOr<Value> maybeBuffer =
176 getBuffer(rewriter, value: collapseShapeOp.getSrc(), options, state);
177 if (failed(Result: maybeBuffer))
178 return failure();
179 Value buffer = *maybeBuffer;
180 auto bufferType = cast<MemRefType>(Val: buffer.getType());
181
182 if (tensorResultType.getRank() == 0) {
183 // 0-d collapses must go through a different op builder.
184 MemRefType resultType;
185
186 if (bufferType.getLayout().isIdentity()) {
187 // Standard layout: result type has no offset.
188 MemRefLayoutAttrInterface layout;
189 resultType = MemRefType::get(shape: {}, elementType: tensorResultType.getElementType(),
190 layout, memorySpace: bufferType.getMemorySpace());
191 } else {
192 // Source memref has a layout map: result type has the same offset as
193 // the source type.
194 SmallVector<int64_t> strides;
195 int64_t offset;
196 if (failed(Result: bufferType.getStridesAndOffset(strides, offset)))
197 return failure();
198 resultType = MemRefType::get(
199 shape: {}, elementType: tensorResultType.getElementType(),
200 layout: StridedLayoutAttr::get(context: op->getContext(), offset, strides: {}),
201 memorySpace: bufferType.getMemorySpace());
202 }
203
204 replaceOpWithNewBufferizedOp<memref::CollapseShapeOp>(
205 rewriter, op, args&: resultType, args&: buffer, args: collapseShapeOp.getReassociation());
206 return success();
207 }
208
209 // If the dims are not collapsible (due to an incompatible source layout
210 // map), force an out-of-place bufferization, i.e., a buffer copy. This
211 // newly allocated buffer will have no layout map and thus be collapsible.
212 bool canBeCollapsed = memref::CollapseShapeOp::isGuaranteedCollapsible(
213 srcType: bufferType, reassociation: collapseShapeOp.getReassociationIndices());
214 if (!canBeCollapsed) {
215 // TODO: Create alloc_tensor ops during TensorCopyInsertion.
216 AnalysisState analysisState(options);
217 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
218 b&: rewriter, loc: op->getLoc(), shapedValue: collapseShapeOp.getSrc(), options, state);
219 if (failed(Result: tensorAlloc))
220 return failure();
221 auto memrefType =
222 MemRefType::get(shape: collapseShapeOp.getSrcType().getShape(),
223 elementType: collapseShapeOp.getSrcType().getElementType(),
224 map: AffineMap(), memorySpace: bufferType.getMemorySpace());
225 buffer = bufferization::ToBufferOp::create(builder&: rewriter, location: op->getLoc(),
226 buffer: memrefType, tensor: *tensorAlloc);
227 }
228
229 // Result type is inferred by the builder.
230 replaceOpWithNewBufferizedOp<memref::CollapseShapeOp>(
231 rewriter, op, args&: buffer, args: collapseShapeOp.getReassociationIndices());
232 return success();
233 }
234};
235
236/// Bufferization of tensor.dim. Replace with memref.dim.
237struct DimOpInterface
238 : public BufferizableOpInterface::ExternalModel<DimOpInterface,
239 tensor::DimOp> {
240 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
241 const AnalysisState &state) const {
242 // The op reads the tensor's metadata but not its contents.
243 return false;
244 }
245
246 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
247 const AnalysisState &state) const {
248 return false;
249 }
250
251 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
252 const AnalysisState &state) const {
253 return {};
254 }
255
256 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
257 const BufferizationOptions &options,
258 BufferizationState &state) const {
259 auto dimOp = cast<tensor::DimOp>(Val: op);
260 FailureOr<Value> v = getBuffer(rewriter, value: dimOp.getSource(), options, state);
261 if (failed(Result: v))
262 return failure();
263 replaceOpWithNewBufferizedOp<memref::DimOp>(rewriter, op, args&: *v,
264 args: dimOp.getIndex());
265 return success();
266 }
267};
268
269/// Bufferization of "tensor.empty". Replace with "bufferization.alloc_tensor".
270struct EmptyOpInterface
271 : public BufferizableOpInterface::ExternalModel<EmptyOpInterface,
272 tensor::EmptyOp> {
273 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
274
275 bool resultBufferizesToMemoryWrite(Operation *op, OpResult opResult,
276 const AnalysisState &state) const {
277 // The returned tensor does not have specified contents.
278 return false;
279 }
280
281 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
282 const BufferizationOptions &options,
283 BufferizationState &state) const {
284 auto emptyOp = cast<tensor::EmptyOp>(Val: op);
285
286 // Optimization: Fold away the op if it has no uses.
287 if (op->getUses().empty()) {
288 rewriter.eraseOp(op);
289 return success();
290 }
291
292 // Allocate a tensor. This emits a "bufferization.alloc_tensor" op.
293 FailureOr<Value> allocTensor = allocateTensorForShapedValue(
294 b&: rewriter, loc: op->getLoc(), shapedValue: emptyOp.getResult(), options, state,
295 /*copy=*/false);
296 if (failed(Result: allocTensor))
297 return failure();
298 rewriter.replaceOp(op, newValues: *allocTensor);
299 return success();
300 }
301};
302
303/// Bufferization of tensor.expand_shape. Replace with memref.expand_shape.
304struct ExpandShapeOpInterface
305 : public BufferizableOpInterface::ExternalModel<ExpandShapeOpInterface,
306 tensor::ExpandShapeOp> {
307 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
308 const AnalysisState &state) const {
309 // In contrast to tensor.collapse_shape, this op can always be bufferized
310 // without a copy.
311 return false;
312 }
313
314 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
315 const AnalysisState &state) const {
316 return false;
317 }
318
319 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
320 const AnalysisState &state) const {
321 return {{op->getOpResult(idx: 0), BufferRelation::Equivalent}};
322 }
323
324 FailureOr<BufferLikeType>
325 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
326 const BufferizationState &state,
327 SmallVector<Value> &invocationStack) const {
328 auto expandShapeOp = cast<tensor::ExpandShapeOp>(Val: op);
329 auto maybeSrcBufferType = bufferization::getBufferType(
330 value: expandShapeOp.getSrc(), options, state, invocationStack);
331 if (failed(Result: maybeSrcBufferType))
332 return failure();
333 auto srcBufferType = llvm::cast<MemRefType>(Val&: *maybeSrcBufferType);
334 auto maybeResultType = memref::ExpandShapeOp::computeExpandedType(
335 srcType: srcBufferType, resultShape: expandShapeOp.getResultType().getShape(),
336 reassociation: expandShapeOp.getReassociationIndices());
337 if (failed(Result: maybeResultType))
338 return failure();
339 return cast<BufferLikeType>(Val&: *maybeResultType);
340 }
341
342 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
343 const BufferizationOptions &options,
344 BufferizationState &state) const {
345 auto expandShapeOp = cast<tensor::ExpandShapeOp>(Val: op);
346 FailureOr<BufferLikeType> maybeResultType =
347 bufferization::getBufferType(value: expandShapeOp.getResult(), options, state);
348 if (failed(Result: maybeResultType))
349 return failure();
350 FailureOr<Value> buffer =
351 getBuffer(rewriter, value: expandShapeOp.getSrc(), options, state);
352 if (failed(Result: buffer))
353 return failure();
354
355 auto memrefExpandShape = memref::ExpandShapeOp::create(
356 builder&: rewriter, location: op->getLoc(), resultType: *maybeResultType, src: *buffer,
357 reassociation: expandShapeOp.getReassociationIndices(),
358 outputShape: expandShapeOp.getMixedOutputShape());
359 replaceOpWithBufferizedValues(rewriter, op,
360 values: memrefExpandShape->getResults());
361 return success();
362 }
363};
364
365/// Bufferization of tensor.extract_slice. Replace with memref.subview.
366struct ExtractSliceOpInterface
367 : public BufferizableOpInterface::ExternalModel<ExtractSliceOpInterface,
368 tensor::ExtractSliceOp> {
369 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
370 const AnalysisState &state) const {
371 return false;
372 }
373
374 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
375 const AnalysisState &state) const {
376 return false;
377 }
378
379 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
380 const AnalysisState &state) const {
381 return {{op->getOpResult(idx: 0), BufferRelation::Unknown}};
382 }
383
384 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
385 const BufferizationOptions &options,
386 BufferizationState &state) const {
387 auto extractSliceOp = cast<tensor::ExtractSliceOp>(Val: op);
388 SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets();
389 SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes();
390 SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides();
391 Location loc = extractSliceOp.getLoc();
392
393 // Get source buffer.
394 FailureOr<Value> srcMemref =
395 getBuffer(rewriter, value: extractSliceOp.getSource(), options, state);
396 if (failed(Result: srcMemref))
397 return failure();
398
399 // Take a subview of the source buffer.
400 auto resultMemrefType = bufferization::getBufferType(
401 value: extractSliceOp.getResult(), options, state);
402 if (failed(Result: resultMemrefType))
403 return failure();
404 Value subView = memref::SubViewOp::create(
405 builder&: rewriter, location: loc, resultType: llvm::cast<MemRefType>(Val&: *resultMemrefType), source: *srcMemref,
406 offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides);
407
408 replaceOpWithBufferizedValues(rewriter, op, values: subView);
409 return success();
410 }
411
412 FailureOr<BufferLikeType>
413 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
414 const BufferizationState &state,
415 SmallVector<Value> &invocationStack) const {
416 auto extractSliceOp = cast<tensor::ExtractSliceOp>(Val: op);
417 assert(value == extractSliceOp.getResult() && "invalid value");
418 auto srcMemrefType = bufferization::getBufferType(
419 value: extractSliceOp.getSource(), options, state, invocationStack);
420 if (failed(Result: srcMemrefType))
421 return failure();
422 SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets();
423 SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes();
424 SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides();
425 return cast<BufferLikeType>(Val: memref::SubViewOp::inferRankReducedResultType(
426 resultShape: extractSliceOp.getType().getShape(),
427 sourceMemRefType: llvm::cast<MemRefType>(Val&: *srcMemrefType), staticOffsets: mixedOffsets, staticSizes: mixedSizes,
428 staticStrides: mixedStrides));
429 }
430};
431
432/// Bufferization of tensor.extract. Replace with memref.load.
433struct ExtractOpInterface
434 : public BufferizableOpInterface::ExternalModel<ExtractOpInterface,
435 tensor::ExtractOp> {
436 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
437 const AnalysisState &state) const {
438 return true;
439 }
440
441 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
442 const AnalysisState &state) const {
443 return false;
444 }
445
446 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
447 const AnalysisState &state) const {
448 return {};
449 }
450
451 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
452 const BufferizationOptions &options,
453 BufferizationState &state) const {
454 auto extractOp = cast<tensor::ExtractOp>(Val: op);
455 FailureOr<Value> srcMemref =
456 getBuffer(rewriter, value: extractOp.getTensor(), options, state);
457 if (failed(Result: srcMemref))
458 return failure();
459 replaceOpWithNewBufferizedOp<memref::LoadOp>(rewriter, op, args&: *srcMemref,
460 args: extractOp.getIndices());
461 return success();
462 }
463};
464
465// Implements backtracking to traverse indices of the output buffer while
466// iterating over op.elements().
467static void createStores(RewriterBase &rewriter, Location loc, int dim,
468 Value buffer, ArrayRef<int64_t> shape,
469 ArrayRef<Value> constants,
470 OperandRange::iterator &elementIt,
471 SmallVectorImpl<Value> &indices) {
472 if (dim == static_cast<int>(shape.size()) - 1) {
473 for (int i = 0; i < shape.back(); ++i) {
474 indices.back() = constants[i];
475 memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: *elementIt, memref: buffer, indices);
476 ++elementIt;
477 }
478 return;
479 }
480 for (int i = 0; i < shape[dim]; ++i) {
481 indices[dim] = constants[i];
482 createStores(rewriter, loc, dim: dim + 1, buffer, shape, constants, elementIt,
483 indices);
484 }
485}
486
487/// Bufferization of tensor.from_elements.
488struct FromElementsOpInterface
489 : public BufferizableOpInterface::ExternalModel<FromElementsOpInterface,
490 tensor::FromElementsOp> {
491
492 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
493
494 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
495 const BufferizationOptions &options,
496 BufferizationState &state) const {
497 auto fromElementsOp = cast<tensor::FromElementsOp>(Val: op);
498 auto tensorType = cast<RankedTensorType>(Val: fromElementsOp.getType());
499
500 // Allocate a buffer for the result.
501 Location loc = op->getLoc();
502 auto shape = tensorType.getShape();
503 // TODO: Create alloc_tensor ops during TensorCopyInsertion.
504 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
505 b&: rewriter, loc, shapedValue: fromElementsOp.getResult(), options, state,
506 /*copy=*/false);
507 if (failed(Result: tensorAlloc))
508 return failure();
509 FailureOr<BufferLikeType> memrefType =
510 bufferization::getBufferType(value: *tensorAlloc, options, state);
511 if (failed(Result: memrefType))
512 return failure();
513 Value buffer = bufferization::ToBufferOp::create(builder&: rewriter, location: op->getLoc(),
514 buffer: *memrefType, tensor: *tensorAlloc);
515
516 // Case: tensor<0xelem_type>.
517 if (fromElementsOp.getElements().empty()) {
518 replaceOpWithBufferizedValues(rewriter, op, values: buffer);
519 return success();
520 }
521
522 // Case: tensor<elem_type>.
523 if (shape.empty()) {
524 memref::StoreOp::create(builder&: rewriter, location: loc,
525 valueToStore: fromElementsOp.getElements().front(), memref: buffer);
526 replaceOpWithBufferizedValues(rewriter, op, values: buffer);
527 return success();
528 }
529
530 // Create constants for the range of possible indices [0, max{shape_i}).
531 auto maxDim = *llvm::max_element(Range&: shape);
532 SmallVector<Value, 2> constants;
533 constants.reserve(N: maxDim);
534 for (int i = 0; i < maxDim; ++i)
535 constants.push_back(Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i));
536
537 // Traverse all `elements` and create `memref.store` ops.
538 auto elementIt = fromElementsOp.getElements().begin();
539 SmallVector<Value, 2> indices(tensorType.getRank(), constants[0]);
540 createStores(rewriter, loc, /*dim=*/0, buffer, shape, constants, elementIt,
541 indices);
542
543 replaceOpWithBufferizedValues(rewriter, op, values: buffer);
544
545 return success();
546 }
547};
548
549/// Lower the body of a tensor.generate like op (one index-typed bbArg per dim).
550/// Such ops are lowered to linalg.map with the given tensor as a destination.
551///
552/// Example:
553/// ```
554/// %r = tensor.generate %x, %y {
555/// ^bb0(%arg0: index, %arg1: index):
556/// %0 = "some_op"(%arg0, %arg1) : (index, index) -> (index)
557/// tensor.yield %0 : index
558/// } : tensor<?x?xindex>
559/// ```
560///
561/// Is lowered to:
562/// ```
563/// linalg.map ins() outs(%dest) {
564/// %d0 = linalg.index 0 : index
565/// %d1 = linalg.index 1 : index
566/// %0 = "some_op"(%d0, %d1) : (index, index) -> (index)
567/// linalg.yield %0 : index
568/// }
569/// ```
570static Value lowerGenerateLikeOpBody(RewriterBase &rewriter, Location loc,
571 Value tensorDestination,
572 ValueRange dynamicSizes,
573 Region &generateBody) {
574 assert(generateBody.hasOneBlock() && "expected body with single block");
575 auto tensorType = cast<RankedTensorType>(Val: tensorDestination.getType());
576 assert(generateBody.getNumArguments() == tensorType.getRank() &&
577 "rank mismatch");
578
579 // Create linalg::MapOp.
580 OpBuilder::InsertionGuard g(rewriter);
581 auto linalgOp =
582 linalg::MapOp::create(builder&: rewriter, location: loc, result: tensorType, /*inputs=*/ValueRange(),
583 /*init=*/tensorDestination);
584 Block &linalgBody = linalgOp.getMapper().emplaceBlock();
585 linalgBody.addArgument(type: tensorType.getElementType(), loc);
586
587 // Create linalg::IndexOps.
588 rewriter.setInsertionPointToStart(&linalgBody);
589 SmallVector<Value> indices;
590 for (int64_t dim = 0; dim < tensorType.getRank(); ++dim)
591 indices.push_back(Elt: linalg::IndexOp::create(builder&: rewriter, location: loc, dim));
592
593 // Move over body.
594 rewriter.mergeBlocks(source: &generateBody.front(), dest: &linalgBody, argValues: indices);
595 auto yieldOp = cast<tensor::YieldOp>(Val: linalgBody.getTerminator());
596 rewriter.replaceOpWithNewOp<linalg::YieldOp>(op: yieldOp, args: yieldOp.getValue());
597
598 return linalgOp.getResult()[0];
599}
600
601/// Bufferization of tensor.generate.
602struct GenerateOpInterface
603 : public BufferizableOpInterface::ExternalModel<GenerateOpInterface,
604 tensor::GenerateOp> {
605
606 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
607
608 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
609 const BufferizationOptions &options,
610 BufferizationState &state) const {
611 auto generateOp = cast<tensor::GenerateOp>(Val: op);
612
613 auto type = generateOp.getResult().getType();
614
615 // TODO: Implement memory space for this op.
616 if (options.defaultMemorySpaceFn(cast<TensorLikeType>(Val&: type)) != Attribute())
617 return op->emitError(message: "memory space not implemented yet");
618
619 // Allocate memory.
620 Location loc = op->getLoc();
621 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
622 b&: rewriter, loc, shapedValue: generateOp.getResult(), options, state,
623 /*copy=*/false);
624 if (failed(Result: tensorAlloc))
625 return failure();
626
627 Value result = lowerGenerateLikeOpBody(rewriter, loc, tensorDestination: *tensorAlloc,
628 dynamicSizes: generateOp.getDynamicExtents(),
629 generateBody&: generateOp.getBody());
630 rewriter.replaceOp(op: generateOp, newValues: result);
631
632 return success();
633 }
634};
635
636/// Bufferization of tensor.insert. Replace with memref.store.
637///
638/// Note: DstBufferizableOpInterfaceExternalModel provides many default method
639/// implementations for DestinationStyle ops.
640struct InsertOpInterface
641 : public DstBufferizableOpInterfaceExternalModel<InsertOpInterface,
642 tensor::InsertOp> {
643 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
644 const BufferizationOptions &options,
645 BufferizationState &state) const {
646 auto insertOp = cast<tensor::InsertOp>(Val: op);
647 FailureOr<Value> destMemref =
648 getBuffer(rewriter, value: insertOp.getDest(), options, state);
649 if (failed(Result: destMemref))
650 return failure();
651 memref::StoreOp::create(builder&: rewriter, location: insertOp.getLoc(), valueToStore: insertOp.getScalar(),
652 memref: *destMemref, indices: insertOp.getIndices());
653 replaceOpWithBufferizedValues(rewriter, op, values: *destMemref);
654 return success();
655 }
656};
657
658template <typename InsertOpTy>
659static bool insertSliceOpRequiresRead(InsertOpTy insertSliceOp,
660 OpOperand &opOperand) {
661 // The source is always read.
662 if (opOperand == insertSliceOp.getSourceMutable())
663 return true;
664
665 // For the destination, it depends...
666 assert(opOperand == insertSliceOp.getDestMutable() && "expected dest");
667
668 // Dest is not read if it is entirely overwritten. E.g.:
669 // tensor.insert_slice %a into %t[0][10][1] : ... into tensor<10xf32>
670 bool allOffsetsZero =
671 llvm::all_of(insertSliceOp.getMixedOffsets(), isZeroInteger);
672 RankedTensorType destType = insertSliceOp.getDestType();
673 bool sizesMatchDestSizes =
674 areConstantIntValues(insertSliceOp.getMixedSizes(), destType.getShape());
675 bool allStridesOne =
676 areAllConstantIntValue(insertSliceOp.getMixedStrides(), 1);
677 return !(allOffsetsZero && sizesMatchDestSizes && allStridesOne);
678}
679
680/// Bufferization of tensor.insert_slice. Replace with a memory copy. Under
681/// certain circumstances, this op can also be a no-op.
682///
683/// Note: DstBufferizableOpInterfaceExternalModel provides many default method
684/// implementations for DestinationStyle ops.
685struct InsertSliceOpInterface
686 : public DstBufferizableOpInterfaceExternalModel<InsertSliceOpInterface,
687 tensor::InsertSliceOp> {
688 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
689 const AnalysisState &state) const {
690 return insertSliceOpRequiresRead(insertSliceOp: cast<tensor::InsertSliceOp>(Val: op),
691 opOperand);
692 }
693
694 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
695 const BufferizationOptions &options,
696 BufferizationState &state) const {
697 // insert_slice ops arise from tiling and bufferizing them out-of-place is
698 // generally a deal breaker. When used with loops, this ends up cloning the
699 // whole tensor on every single iteration and is a symptom of a
700 // catastrophically bad scheduling decision.
701 // TODO: be very loud about it or even consider failing the pass.
702 auto insertSliceOp = cast<tensor::InsertSliceOp>(Val: op);
703 SmallVector<OpFoldResult> mixedOffsets = insertSliceOp.getMixedOffsets();
704 SmallVector<OpFoldResult> mixedSizes = insertSliceOp.getMixedSizes();
705 SmallVector<OpFoldResult> mixedStrides = insertSliceOp.getMixedStrides();
706 Location loc = insertSliceOp.getLoc();
707
708 // Get destination buffer.
709 FailureOr<Value> dstMemref =
710 getBuffer(rewriter, value: insertSliceOp.getDest(), options, state);
711 if (failed(Result: dstMemref))
712 return failure();
713
714 // Take a subview of the destination buffer.
715 auto dstMemrefType = cast<MemRefType>(Val: dstMemref->getType());
716 MemRefType subviewMemRefType =
717 memref::SubViewOp::inferRankReducedResultType(
718 resultShape: insertSliceOp.getSourceType().getShape(), sourceMemRefType: dstMemrefType,
719 staticOffsets: mixedOffsets, staticSizes: mixedSizes, staticStrides: mixedStrides);
720 Value subView =
721 memref::SubViewOp::create(builder&: rewriter, location: loc, resultType: subviewMemRefType, source: *dstMemref,
722 offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides);
723
724 // Copy tensor. If this tensor.insert_slice has a matching
725 // tensor.extract_slice, the copy operation will eventually fold away.
726 FailureOr<Value> srcMemref =
727 getBuffer(rewriter, value: insertSliceOp.getSource(), options, state);
728 if (failed(Result: srcMemref))
729 return failure();
730 if (failed(Result: options.memCpyFn(rewriter, loc, *srcMemref, subView)))
731 return failure();
732
733 replaceOpWithBufferizedValues(rewriter, op, values: *dstMemref);
734 return success();
735 }
736};
737
738/// Bufferization of tensor.pad. Replace with bufferization.alloc_tensor +
739/// linalg.map + insert_slice.
740/// For best performance, vectorize before bufferization (better performance in
741/// case of padding with a constant).
742struct PadOpInterface
743 : public BufferizableOpInterface::ExternalModel<PadOpInterface,
744 tensor::PadOp> {
745 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
746
747 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
748 const AnalysisState &state) const {
749 return true;
750 }
751
752 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
753 const AnalysisState &state) const {
754 return false;
755 }
756
757 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
758 const AnalysisState &state) const {
759 return {};
760 }
761
762 FailureOr<BufferLikeType>
763 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
764 const BufferizationState &state,
765 SmallVector<Value> &invocationStack) const {
766 // Infer memory space from the source tensor.
767 auto padOp = cast<tensor::PadOp>(Val: op);
768 auto maybeSrcBufferType =
769 bufferization::detail::asMemRefType(bufferType: bufferization::getBufferType(
770 value: padOp.getSource(), options, state, invocationStack));
771 if (failed(Result: maybeSrcBufferType))
772 return failure();
773 MemRefLayoutAttrInterface layout;
774 return cast<BufferLikeType>(
775 Val: MemRefType::get(shape: padOp.getResultType().getShape(),
776 elementType: padOp.getResultType().getElementType(), layout,
777 memorySpace: maybeSrcBufferType->getMemorySpace()));
778 }
779
780 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
781 const BufferizationOptions &options,
782 BufferizationState &state) const {
783 auto padOp = cast<tensor::PadOp>(Val: op);
784 Location loc = padOp.getLoc();
785 RankedTensorType resultType = padOp.getResultType();
786 RankedTensorType srcType = padOp.getSourceType();
787
788 auto toValue = [&](OpFoldResult ofr) {
789 if (auto value = dyn_cast<Value>(Val&: ofr))
790 return value;
791 return arith::ConstantIndexOp::create(builder&: rewriter, location: loc,
792 value: *getConstantIntValue(ofr))
793 .getResult();
794 };
795
796 // Compute dynamic result dimensions.
797 SmallVector<OpFoldResult> mixedLowPad = padOp.getMixedLowPad();
798 SmallVector<OpFoldResult> mixedHighPad = padOp.getMixedHighPad();
799 SmallVector<Value> dynamicSizes;
800 for (int64_t i = 0; i < resultType.getRank(); ++i) {
801 if (!resultType.isDynamicDim(idx: i))
802 continue;
803 Value srcDim = tensor::DimOp::create(builder&: rewriter, location: loc, source: padOp.getSource(), index: i);
804 Value lowPad = toValue(mixedLowPad[i]);
805 Value highPad = toValue(mixedHighPad[i]);
806 AffineExpr s0, s1, s2;
807 bindSymbols(ctx: op->getContext(), exprs&: s0, exprs&: s1, exprs&: s2);
808 AffineExpr sumExpr = s0 + s1 + s2;
809 Value sum = affine::AffineApplyOp::create(
810 builder&: rewriter, location: loc, exprList: sumExpr, mapOperands: ValueRange{srcDim, lowPad, highPad});
811 dynamicSizes.push_back(Elt: sum);
812 }
813
814 // Allocate a buffer for the padded result.
815 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
816 b&: rewriter, loc, shapedValue: padOp.getResult(), options, state,
817 /*copy=*/false);
818 if (failed(Result: tensorAlloc))
819 return failure();
820
821 // tensor::PadOp is like tensor::GenerateOp: The only difference is that
822 // only a part of the generated tensor is needed. For simplicity, we reuse
823 // the same functionality here.
824 Value filledBuffer = lowerGenerateLikeOpBody(
825 rewriter, loc, tensorDestination: *tensorAlloc, dynamicSizes, generateBody&: padOp.getBodyRegion());
826
827 // Create tensor::InsertSliceOp.
828 SmallVector<OpFoldResult> sliceSizes =
829 getMixedSizes(builder&: rewriter, loc, value: padOp.getSource());
830 SmallVector<OpFoldResult> sliceStrides(srcType.getRank(),
831 rewriter.getIndexAttr(value: 1));
832 rewriter.replaceOpWithNewOp<tensor::InsertSliceOp>(
833 op: padOp, args: padOp.getSource(), args&: filledBuffer,
834 /*offsets=*/args: padOp.getMixedLowPad(), args&: sliceSizes, args&: sliceStrides);
835
836 return success();
837 }
838};
839
840/// Bufferization of tensor.rank. Replace with memref.rank.
841struct RankOpInterface
842 : public BufferizableOpInterface::ExternalModel<RankOpInterface,
843 tensor::RankOp> {
844 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
845 const AnalysisState &state) const {
846 // The op reads the tensor's metadata but not its contents.
847 return false;
848 }
849
850 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
851 const AnalysisState &state) const {
852 return false;
853 }
854
855 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
856 const AnalysisState &state) const {
857 return {};
858 }
859
860 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
861 const BufferizationOptions &options,
862 BufferizationState &state) const {
863 auto rankOp = cast<tensor::RankOp>(Val: op);
864 FailureOr<Value> v =
865 getBuffer(rewriter, value: rankOp.getTensor(), options, state);
866 if (failed(Result: v))
867 return failure();
868 replaceOpWithNewBufferizedOp<memref::RankOp>(rewriter, op, args: rankOp.getType(),
869 args&: *v);
870 return success();
871 }
872};
873
874/// Bufferization of tensor.reshape. Replace with memref.reshape.
875struct ReshapeOpInterface
876 : public BufferizableOpInterface::ExternalModel<ReshapeOpInterface,
877 tensor::ReshapeOp> {
878 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
879 const AnalysisState &state) const {
880 // Depending on the layout map, the source buffer may have to be copied.
881 auto reshapeOp = cast<tensor::ReshapeOp>(Val: op);
882 return opOperand == reshapeOp.getShapeMutable();
883 }
884
885 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
886 const AnalysisState &state) const {
887 return false;
888 }
889
890 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
891 const AnalysisState &state) const {
892 // Only the 'source' operand aliases the result.
893 auto reshapeOp = cast<tensor::ReshapeOp>(Val: op);
894 if (reshapeOp.getSourceMutable() != opOperand)
895 return {};
896 return {{op->getOpResult(idx: 0), BufferRelation::Equivalent}};
897 }
898
899 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
900 const BufferizationOptions &options,
901 BufferizationState &state) const {
902 auto reshapeOp = cast<tensor::ReshapeOp>(Val: op);
903 FailureOr<Value> srcBuffer =
904 getBuffer(rewriter, value: reshapeOp.getSource(), options, state);
905 FailureOr<Value> shapeBuffer =
906 getBuffer(rewriter, value: reshapeOp.getShape(), options, state);
907 if (failed(Result: srcBuffer) || failed(Result: shapeBuffer))
908 return failure();
909 auto maybeResultMemRefType =
910 bufferization::getBufferType(value: reshapeOp.getResult(), options, state);
911 if (failed(Result: maybeResultMemRefType))
912 return failure();
913
914 // memref.reshape requires the source buffer to have an identity layout.
915 // If the source memref does not have an identity layout, copy the source
916 // into a new buffer with an identity layout.
917 auto srcType = llvm::dyn_cast<MemRefType>(Val: srcBuffer->getType());
918 if (srcType && !srcType.getLayout().isIdentity()) {
919 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
920 b&: rewriter, loc: op->getLoc(), shapedValue: reshapeOp.getSource(), options, state);
921 if (failed(Result: tensorAlloc))
922 return failure();
923 auto memrefType = MemRefType::get(
924 shape: srcType.getShape(), elementType: srcType.getElementType(), map: AffineMap(),
925 memorySpace: cast<BaseMemRefType>(Val: srcBuffer->getType()).getMemorySpace());
926 srcBuffer = bufferization::ToBufferOp::create(builder&: rewriter, location: op->getLoc(),
927 buffer: memrefType, tensor: *tensorAlloc)
928 .getResult();
929 }
930
931 replaceOpWithNewBufferizedOp<memref::ReshapeOp>(
932 rewriter, op, args&: maybeResultMemRefType.value(), args&: *srcBuffer, args&: *shapeBuffer);
933 return success();
934 }
935
936 FailureOr<BufferLikeType>
937 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
938 const BufferizationState &state,
939 SmallVector<Value> &invocationStack) const {
940 auto reshapeOp = cast<tensor::ReshapeOp>(Val: op);
941 assert(value == reshapeOp.getResult() && "unexpected value provided");
942 auto maybeSourceBufferType = bufferization::getBufferType(
943 value: reshapeOp.getSource(), options, state, invocationStack);
944 if (failed(Result: maybeSourceBufferType))
945 return failure();
946 return cast<BufferLikeType>(Val: getMemRefTypeWithStaticIdentityLayout(
947 tensorType: reshapeOp.getResult().getType(),
948 memorySpace: cast<BaseMemRefType>(Val&: maybeSourceBufferType.value()).getMemorySpace()));
949 }
950};
951
952/// Analysis of ParallelInsertSliceOp.
953struct ParallelInsertSliceOpInterface
954 : public BufferizableOpInterface::ExternalModel<
955 ParallelInsertSliceOpInterface, ParallelInsertSliceOp> {
956 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
957 const AnalysisState &state) const {
958 return {};
959 }
960
961 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
962 const AnalysisState &state) const {
963 return opOperand == cast<ParallelInsertSliceOp>(Val: op).getSourceMutable();
964 }
965
966 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
967 const AnalysisState &state) const {
968 auto parallelInsertSliceOp = cast<ParallelInsertSliceOp>(Val: op);
969 return opOperand == parallelInsertSliceOp.getDestMutable();
970 }
971
972 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
973 const BufferizationOptions &options,
974 BufferizationState &state) const {
975 OpBuilder::InsertionGuard g(rewriter);
976 auto parallelInsertSliceOp = cast<ParallelInsertSliceOp>(Val: op);
977 InParallelOpInterface parallelCombiningParent =
978 parallelInsertSliceOp.getParallelCombiningParent();
979
980 // Bufferize the op outside of the in parallel terminator.
981 rewriter.setInsertionPoint(parallelCombiningParent);
982
983 // Get source and destination buffers.
984 FailureOr<Value> destBuffer =
985 getBuffer(rewriter, value: parallelInsertSliceOp.getDest(), options, state);
986 if (failed(Result: destBuffer))
987 return failure();
988 FailureOr<Value> srcBuffer =
989 getBuffer(rewriter, value: parallelInsertSliceOp.getSource(), options, state);
990 if (failed(Result: srcBuffer))
991 return failure();
992
993 // Take a subview of the destination buffer.
994 auto destBufferType = cast<MemRefType>(Val: destBuffer->getType());
995 MemRefType subviewMemRefType =
996 memref::SubViewOp::inferRankReducedResultType(
997 resultShape: parallelInsertSliceOp.getSourceType().getShape(), sourceMemRefType: destBufferType,
998 staticOffsets: parallelInsertSliceOp.getMixedOffsets(),
999 staticSizes: parallelInsertSliceOp.getMixedSizes(),
1000 staticStrides: parallelInsertSliceOp.getMixedStrides());
1001 Value subview = memref::SubViewOp::create(
1002 builder&: rewriter, location: parallelInsertSliceOp.getLoc(), resultType: subviewMemRefType,
1003 source: *destBuffer, offsets: parallelInsertSliceOp.getMixedOffsets(),
1004 sizes: parallelInsertSliceOp.getMixedSizes(),
1005 strides: parallelInsertSliceOp.getMixedStrides());
1006
1007 // This memcpy will fold away if everything bufferizes in-place.
1008 if (failed(Result: options.memCpyFn(rewriter, parallelInsertSliceOp.getLoc(),
1009 *srcBuffer, subview)))
1010 return failure();
1011
1012 // In case the source was allocated in the same block, make sure that the
1013 // deallocation op (if any) appears after the memcpy. By default, deallocs
1014 // are placed before the terminator, but this does not work for ForallOp
1015 // because the terminator does more than just yielding a value.
1016 //
1017 // Note: This is not a problem for the destination buffer because these are
1018 // assumed to always bufferize in-place.
1019 for (Operation *user : srcBuffer->getUsers()) {
1020 if (hasEffect<MemoryEffects::Free>(op: user)) {
1021 if (user->getBlock() == parallelCombiningParent->getBlock())
1022 rewriter.moveOpBefore(op: user, existingOp: user->getBlock()->getTerminator());
1023 break;
1024 }
1025 }
1026
1027 // Delete the op.
1028 rewriter.eraseOp(op);
1029 return success();
1030 }
1031
1032 /// tensor.parallel_insert_slice op has implicit inplace behavior. We
1033 /// shouldn't create copy to resolve conflict.
1034 LogicalResult
1035 resolveConflicts(Operation *op, RewriterBase &rewriter,
1036 const AnalysisState &analysisState,
1037 const BufferizationState &bufferizationState) const {
1038 return success();
1039 }
1040};
1041
1042/// Bufferization of tensor.splat. Bufferizes to a new allocation that is filled
1043/// with a linalg.map. Similar to tensor.generate.
1044struct SplatOpInterface
1045 : public BufferizableOpInterface::ExternalModel<SplatOpInterface,
1046 tensor::SplatOp> {
1047
1048 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
1049
1050 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
1051 const BufferizationOptions &options,
1052 BufferizationState &state) const {
1053 OpBuilder::InsertionGuard g(rewriter);
1054 auto splatOp = cast<tensor::SplatOp>(Val: op);
1055
1056 // Allocate memory.
1057 Location loc = op->getLoc();
1058 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
1059 b&: rewriter, loc, shapedValue: splatOp.getResult(), options, state,
1060 /*copy=*/false);
1061 if (failed(Result: tensorAlloc))
1062 return failure();
1063
1064 // Create linalg::MapOp.
1065 auto tensorType = cast<RankedTensorType>(Val: tensorAlloc->getType());
1066
1067 // TODO: Implement memory space for this op.
1068 if (options.defaultMemorySpaceFn(cast<TensorLikeType>(Val&: tensorType)) !=
1069 Attribute())
1070 return op->emitError(message: "memory space not implemented yet");
1071
1072 auto linalgOp = linalg::MapOp::create(builder&: rewriter, location: loc, result: tensorType,
1073 /*inputs=*/ValueRange(),
1074 /*init=*/*tensorAlloc);
1075 Block &linalgBody = linalgOp.getMapper().emplaceBlock();
1076 linalgBody.addArgument(type: tensorType.getElementType(), loc);
1077
1078 // Create linalg::IndexOps.
1079 rewriter.setInsertionPointToStart(&linalgBody);
1080 linalg::YieldOp::create(builder&: rewriter, location: loc, values: splatOp.getInput());
1081 rewriter.replaceOp(op: splatOp, newValues: linalgOp.getResult()[0]);
1082
1083 return success();
1084 }
1085};
1086
1087/// Bufferization of tensor.concat. Bufferizes to a new allocation that is
1088/// filled with copy ops. Similar to tensor.from_elements, but using memref.copy
1089/// on subviews instead of memref.store.
1090struct ConcatOpInterface
1091 : public BufferizableOpInterface::ExternalModel<ConcatOpInterface,
1092 tensor::ConcatOp> {
1093
1094 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
1095
1096 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
1097 const AnalysisState &state) const {
1098 return false;
1099 }
1100
1101 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
1102 const AnalysisState &state) const {
1103 return true;
1104 }
1105
1106 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
1107 const AnalysisState &state) const {
1108 return {};
1109 }
1110
1111 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
1112 const BufferizationOptions &options,
1113 BufferizationState &state) const {
1114 OpBuilder::InsertionGuard g(rewriter);
1115 auto concatOp = cast<tensor::ConcatOp>(Val: op);
1116
1117 // Allocate memory.
1118 Location loc = op->getLoc();
1119 FailureOr<Value> tensorAlloc = allocateTensorForShapedValue(
1120 b&: rewriter, loc, shapedValue: concatOp.getResult(), options, state,
1121 /*copy=*/false);
1122 if (failed(Result: tensorAlloc))
1123 return failure();
1124 auto tensorType = cast<RankedTensorType>(Val: tensorAlloc->getType());
1125 FailureOr<BufferLikeType> memrefType =
1126 bufferization::getBufferType(value: *tensorAlloc, options, state);
1127 if (failed(Result: memrefType))
1128 return failure();
1129 Value dstBuffer = bufferization::ToBufferOp::create(
1130 builder&: rewriter, location: op->getLoc(), buffer: *memrefType, tensor: *tensorAlloc);
1131
1132 // Extract the dimension for the concat op
1133 uint64_t concatDim = concatOp.getDim();
1134
1135 SmallVector<OpFoldResult> offsets(tensorType.getRank(),
1136 rewriter.getIndexAttr(value: 0));
1137 SmallVector<OpFoldResult> strides(tensorType.getRank(),
1138 rewriter.getIndexAttr(value: 1));
1139 SmallVector<OpFoldResult> sizes =
1140 memref::getMixedSizes(builder&: rewriter, loc, value: dstBuffer);
1141
1142 AffineExpr s0, s1;
1143 bindSymbols(ctx: rewriter.getContext(), exprs&: s0, exprs&: s1);
1144 auto sum = [&](OpFoldResult v1, OpFoldResult v2) {
1145 return affine::makeComposedFoldedAffineApply(b&: rewriter, loc, expr: s0 + s1,
1146 operands: {v1, v2});
1147 };
1148
1149 OpFoldResult concatDimOffset = rewriter.getIndexAttr(value: 0);
1150 for (auto operand : concatOp.getInputs()) {
1151 // Get the buffer for the operand.
1152 FailureOr<Value> srcBuffer = getBuffer(rewriter, value: operand, options, state);
1153 if (failed(Result: srcBuffer))
1154 return failure();
1155
1156 // Each operand may have a different size along the concat dimension,
1157 // so the offset on that axis must accumulate through the loop, and the
1158 // size must change to the size of the current operand.
1159 auto operandTensorType = cast<RankedTensorType>(Val: operand.getType());
1160 offsets[concatDim] = concatDimOffset;
1161 OpFoldResult concatDimSize =
1162 memref::getMixedSize(builder&: rewriter, loc, value: *srcBuffer, dim: concatDim);
1163 sizes[concatDim] = concatDimSize;
1164
1165 // Create a subview of the destination buffer.
1166 auto dstMemrefType = cast<MemRefType>(Val&: *memrefType);
1167 MemRefType subviewMemRefType =
1168 memref::SubViewOp::inferRankReducedResultType(
1169 resultShape: operandTensorType.getShape(), sourceMemRefType: dstMemrefType, staticOffsets: offsets, staticSizes: sizes,
1170 staticStrides: strides);
1171 Value subview = memref::SubViewOp::create(
1172 builder&: rewriter, location: loc, resultType: subviewMemRefType, source: dstBuffer, offsets, sizes, strides);
1173
1174 // Copy the source buffer into the destination subview.
1175 if (failed(Result: options.memCpyFn(rewriter, loc, *srcBuffer, subview)))
1176 return failure();
1177
1178 concatDimOffset = sum(concatDimOffset, concatDimSize);
1179 }
1180
1181 replaceOpWithBufferizedValues(rewriter, op, values: dstBuffer);
1182 return success();
1183 }
1184};
1185
1186} // namespace
1187} // namespace tensor
1188} // namespace mlir
1189
1190void mlir::tensor::registerBufferizableOpInterfaceExternalModels(
1191 DialectRegistry &registry) {
1192 registry.addExtension(extensionFn: +[](MLIRContext *ctx, tensor::TensorDialect *dialect) {
1193 CastOp::attachInterface<CastOpInterface>(context&: *ctx);
1194 CollapseShapeOp::attachInterface<CollapseShapeOpInterface>(context&: *ctx);
1195 ConcatOp::attachInterface<ConcatOpInterface>(context&: *ctx);
1196 DimOp::attachInterface<DimOpInterface>(context&: *ctx);
1197 EmptyOp::attachInterface<EmptyOpInterface>(context&: *ctx);
1198 ExpandShapeOp::attachInterface<ExpandShapeOpInterface>(context&: *ctx);
1199 ExtractSliceOp::attachInterface<ExtractSliceOpInterface>(context&: *ctx);
1200 ExtractOp::attachInterface<ExtractOpInterface>(context&: *ctx);
1201 FromElementsOp::attachInterface<FromElementsOpInterface>(context&: *ctx);
1202 GenerateOp::attachInterface<GenerateOpInterface>(context&: *ctx);
1203 InsertOp::attachInterface<InsertOpInterface>(context&: *ctx);
1204 InsertSliceOp::attachInterface<InsertSliceOpInterface>(context&: *ctx);
1205 PadOp::attachInterface<PadOpInterface>(context&: *ctx);
1206 ParallelInsertSliceOp::attachInterface<ParallelInsertSliceOpInterface>(
1207 context&: *ctx);
1208 RankOp::attachInterface<RankOpInterface>(context&: *ctx);
1209 ReshapeOp::attachInterface<ReshapeOpInterface>(context&: *ctx);
1210 SplatOp::attachInterface<SplatOpInterface>(context&: *ctx);
1211
1212 // Load additional dialects of which ops may get created.
1213 ctx->loadDialect<arith::ArithDialect, linalg::LinalgDialect>();
1214 });
1215
1216 // Bufferization requires SubsetInsertionOpInterface models. Make sure that
1217 // they are registered.
1218 tensor::registerSubsetOpInterfaceExternalModels(registry);
1219}
1220