1//===- ConvertToDestinationStyle.cpp - Convert non-DPS to DPS ops ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains patterns to convert non-DPS ops to DPS ops. New
10// tensor.empty ops are inserted as a destination. Such tensor.empty can be
11// eliminated with "empty tensor elimination", allowing them to bufferize
12// without an allocation (assuming there are no further conflicts).
13//
14//===----------------------------------------------------------------------===//
15//
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
18#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
19#include "mlir/Dialect/Linalg/IR/Linalg.h"
20#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
21#include "mlir/Dialect/Tensor/IR/Tensor.h"
22#include "mlir/Dialect/Utils/StaticValueUtils.h"
23#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
24#include "mlir/IR/Matchers.h"
25#include "mlir/IR/PatternMatch.h"
26#include "llvm/ADT/STLExtras.h"
27
28using namespace mlir;
29using namespace mlir::tensor;
30
31// Implements backtracking to traverse indices of the output buffer while
32// iterating over op.elements().
33static Value createInserts(RewriterBase &rewriter, Location loc, int dim,
34 Value destination, ArrayRef<int64_t> shape,
35 ArrayRef<Value> constants,
36 OperandRange::iterator &elementIt,
37 SmallVectorImpl<Value> &indices) {
38 if (dim == static_cast<int>(shape.size()) - 1) {
39 for (int i = 0; i < shape.back(); ++i) {
40 indices.back() = constants[i];
41 destination = tensor::InsertOp::create(builder&: rewriter, location: loc, scalar: *elementIt,
42 dest: destination, indices);
43 ++elementIt;
44 }
45 return destination;
46 }
47 for (int i = 0; i < shape[dim]; ++i) {
48 indices[dim] = constants[i];
49 destination = createInserts(rewriter, loc, dim: dim + 1, destination, shape,
50 constants, elementIt, indices);
51 }
52 return destination;
53}
54
55/// Create a memcpy from the given source tensor to the given destination
56/// memref. The copy op type can be specified in the `options`.
57static void createMemcpy(OpBuilder &b, Location loc, Value tensorSource,
58 Value memrefDest,
59 const linalg::BufferizeToAllocationOptions &options) {
60 auto tensorType = dyn_cast<RankedTensorType>(Val: tensorSource.getType());
61 assert(tensorType && "expected ranked tensor");
62 assert(isa<MemRefType>(memrefDest.getType()) && "expected ranked memref");
63
64 switch (options.memcpyOp) {
65 case linalg::BufferizeToAllocationOptions::MemcpyOp::
66 MaterializeInDestination: {
67 // Note: This is the preferred way of memcpy'ing because no layout map
68 // and/or memory space must be specified for the source.
69 auto materializeOp = bufferization::MaterializeInDestinationOp::create(
70 builder&: b, location: loc, source: tensorSource, dest: memrefDest);
71 materializeOp.setWritable(true);
72 } break;
73 case linalg::BufferizeToAllocationOptions::MemcpyOp::MemrefCopy: {
74 // TODO: Support custom memory space on source.
75 // We do not know the layout map of the source yet, so use a fully dynamic
76 // layout for best compatibility.
77 Value toBuffer = bufferization::ToBufferOp::create(
78 builder&: b, location: loc, buffer: bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
79 tensor: tensorSource, /*read_only=*/true);
80 memref::CopyOp::create(builder&: b, location: loc, source: toBuffer, target: memrefDest);
81 } break;
82 case linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy: {
83 // TODO: Support custom memory space on source.
84 // We do not know the layout map of the source yet, so use a fully dynamic
85 // layout for best compatibility.
86 Value toBuffer = bufferization::ToBufferOp::create(
87 builder&: b, location: loc, buffer: bufferization::getMemRefTypeWithFullyDynamicLayout(tensorType),
88 tensor: tensorSource, /*read_only=*/true);
89 linalg::CopyOp::create(builder&: b, location: loc, inputs: toBuffer, outputs: memrefDest);
90 } break;
91 };
92}
93
94static Operation *movePaddingToFillOrGenericOp(RewriterBase &rewriter,
95 Location loc, PadOp padOp,
96 Value dest) {
97 OpBuilder::InsertionGuard g(rewriter);
98 RankedTensorType resultType = padOp.getResultType();
99
100 // Collect user/dialect attributes from the pad op to preserve on the newly
101 // created ops.
102 SmallVector<NamedAttribute> preservedAttrs =
103 getPrunedAttributeList(op: padOp, elidedAttrs: PadOp::getAttributeNames());
104
105 // Examine the yielded value to decide if a linalg.generic is needed or a
106 // linalg.fill is sufficient.
107 Value yieldedValue =
108 cast<tensor::YieldOp>(Val: padOp.getBody()->getTerminator()).getValue();
109 Attribute constYieldedValue;
110 // Is the yielded value a bbArg defined outside of the PadOp?
111 bool outsideBbArg =
112 isa<BlockArgument>(Val: yieldedValue) &&
113 cast<BlockArgument>(Val&: yieldedValue).getOwner()->getParentOp() !=
114 padOp.getOperation();
115 // Is the yielded value an OpResult defined outside of the PadOp?
116 bool outsideOpResult =
117 isa<OpResult>(Val: yieldedValue) &&
118 yieldedValue.getDefiningOp()->getParentOp() != padOp.getOperation();
119 bool invariantYieldedValue = outsideBbArg || outsideOpResult;
120 if (matchPattern(value: yieldedValue, pattern: m_Constant(bind_value: &constYieldedValue))) {
121 // Padding with a constant: Create linalg.fill.
122 Dialect *arithDialect =
123 rewriter.getContext()->getLoadedDialect<arith::ArithDialect>();
124 Value fillValue =
125 arithDialect
126 ->materializeConstant(builder&: rewriter, value: constYieldedValue,
127 type: yieldedValue.getType(), loc: yieldedValue.getLoc())
128 ->getResult(idx: 0);
129 auto fillOp = linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange(fillValue),
130 outputs: ValueRange(dest));
131 fillOp->setDiscardableAttrs(preservedAttrs);
132 return fillOp;
133 }
134
135 if (invariantYieldedValue) {
136 // Padding with an invariant value.
137 auto fillOp = linalg::FillOp::create(
138 builder&: rewriter, location: loc, inputs: ValueRange(yieldedValue), outputs: ValueRange(dest));
139 fillOp->setDiscardableAttrs(preservedAttrs);
140 return fillOp;
141 }
142
143 // Create linalg.generic.
144 SmallVector<utils::IteratorType> iteratorTypes(resultType.getRank(),
145 utils::IteratorType::parallel);
146 SmallVector<AffineMap> indexingMaps(
147 1, rewriter.getMultiDimIdentityMap(rank: resultType.getRank()));
148 auto genericOp = linalg::GenericOp::create(
149 builder&: rewriter, location: loc, resultTensorTypes: resultType, /*inputs=*/ValueRange(),
150 /*outputs=*/ValueRange{dest}, /*indexingMaps=*/
151 indexingMaps, iteratorTypes);
152 genericOp->setDiscardableAttrs(preservedAttrs);
153 Block *body = rewriter.createBlock(parent: &genericOp->getRegion(index: 0), insertPt: {},
154 argTypes: resultType.getElementType(), locs: loc);
155 rewriter.setInsertionPointToStart(body);
156 SmallVector<Value> bbArgReplacements;
157 for (int64_t i = 0; i < resultType.getRank(); ++i)
158 bbArgReplacements.push_back(Elt: linalg::IndexOp::create(builder&: rewriter, location: loc, dim: i));
159 rewriter.mergeBlocks(source: padOp.getBody(), dest: body, argValues: bbArgReplacements);
160
161 // Update terminator.
162 auto yieldOp = cast<tensor::YieldOp>(Val: body->getTerminator());
163 rewriter.replaceOpWithNewOp<linalg::YieldOp>(op: yieldOp, args: yieldOp.getValue());
164 return genericOp;
165}
166
167static SmallVector<Value> reifyOrComputeDynamicSizes(OpBuilder &b,
168 Value value) {
169 auto tensorType = cast<RankedTensorType>(Val: value.getType());
170 if (tensorType.hasStaticShape())
171 return {};
172
173 // Try to reify dynamic sizes.
174 ReifiedRankedShapedTypeDims reifiedShape;
175 if (isa<OpResult>(Val: value) &&
176 succeeded(Result: reifyResultShapes(b, op: value.getDefiningOp(), reifiedReturnShapes&: reifiedShape))) {
177 SmallVector<Value> dynSizes;
178 for (int64_t i = 0; i < tensorType.getRank(); ++i) {
179 if (tensorType.isDynamicDim(idx: i))
180 dynSizes.push_back(Elt: cast<Value>(
181 Val&: reifiedShape[cast<OpResult>(Val&: value).getResultNumber()][i]));
182 }
183 return dynSizes;
184 }
185
186 // Create tensor.dim ops.
187 SmallVector<Value> dynSizes;
188 for (int64_t i = 0; i < tensorType.getRank(); ++i) {
189 if (tensorType.isDynamicDim(idx: i))
190 dynSizes.push_back(
191 Elt: DimOp::create(builder&: b, location: value.getLoc(), source: value,
192 index: arith::ConstantIndexOp::create(builder&: b, location: value.getLoc(), value: i)));
193 }
194 return dynSizes;
195}
196
197static Value
198createAllocationForTensor(RewriterBase &rewriter, Location loc, Value value,
199 const linalg::BufferizeToAllocationOptions &options,
200 Attribute memorySpace = {}) {
201 OpBuilder::InsertionGuard g(rewriter);
202 auto tensorType = cast<RankedTensorType>(Val: value.getType());
203
204 // Create buffer allocation.
205 auto memrefType =
206 cast<MemRefType>(Val: bufferization::getMemRefTypeWithStaticIdentityLayout(
207 tensorType, memorySpace));
208 SmallVector<Value> dynamicSizes = reifyOrComputeDynamicSizes(b&: rewriter, value);
209
210 Value alloc;
211 if (options.allocOp ==
212 linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc) {
213 alloc = memref::AllocOp::create(builder&: rewriter, location: loc, memrefType, dynamicSizes);
214 if (options.emitDealloc) {
215 // Place deallocation at the end of the block.
216 rewriter.setInsertionPoint(rewriter.getInsertionBlock()->getTerminator());
217 memref::DeallocOp::create(builder&: rewriter, location: loc, memref: alloc);
218 }
219 } else if (options.allocOp ==
220 linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca) {
221 alloc = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType, dynamicSizes);
222 // No dealloc is needed.
223 }
224
225 return alloc;
226}
227
228Value linalg::bufferizeToAllocation(
229 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
230 PadOp padOp, Attribute memorySpace, Operation *insertionPoint) {
231 // tensor.pad does not have a destination operand.
232 assert(!options.bufferizeDestinationOnly && "invalid options");
233
234 OpBuilder::InsertionGuard g(rewriter);
235 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : padOp);
236 Location loc = padOp.getLoc();
237
238 // Create buffer allocation.
239 Value alloc = createAllocationForTensor(rewriter, loc, value: padOp.getResult(),
240 options, memorySpace);
241 rewriter.setInsertionPoint(padOp);
242
243 if (!padOp.hasZeroLowPad() || !padOp.hasZeroHighPad()) {
244 // Create linalg.fill or linalg.generic. Not needed if there is no padding.
245 Operation *fillOp =
246 movePaddingToFillOrGenericOp(rewriter, loc, padOp, dest: alloc);
247 rewriter.setInsertionPointAfter(fillOp);
248 }
249
250 // Create memcpy.
251 SmallVector<OpFoldResult> sizes =
252 getMixedSizes(builder&: rewriter, loc, value: padOp.getSource());
253 SmallVector<OpFoldResult> strides(padOp.getResultType().getRank(),
254 rewriter.getIndexAttr(value: 1));
255 Value subview = memref::SubViewOp::create(
256 builder&: rewriter, location: loc, source: alloc, /*offsets=*/padOp.getMixedLowPad(), sizes, strides);
257 createMemcpy(b&: rewriter, loc, tensorSource: padOp.getSource(), memrefDest: subview, options);
258
259 // Create bufferization.to_tensor with "restrict" and "writable". The returned
260 // tensor is a new buffer allocation, so it does not alias with any buffer.
261 Value toTensorOp = bufferization::ToTensorOp::create(
262 builder&: rewriter, location: loc, result: padOp.getResult().getType(), buffer: alloc, /*restrict=*/true,
263 /*writable=*/true);
264 rewriter.replaceOp(op: padOp, newValues: toTensorOp);
265 return alloc;
266}
267
268Value linalg::bufferizeToAllocation(
269 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
270 vector::MaskOp maskOp, Attribute memorySpace, Operation *insertionPoint) {
271 assert(llvm::range_size(maskOp.getMaskBlock()->without_terminator()) == 1 &&
272 "expected single masked op");
273 OpBuilder::InsertionGuard g(rewriter);
274
275 // Should the bufferization options and state be function arguments?
276 bufferization::BufferizationOptions bufferizationOptions;
277 bufferization::BufferizationState bufferizationState;
278
279 Operation *yieldOp = maskOp.getMaskRegion().front().getTerminator();
280 assert(isa<vector::YieldOp>(yieldOp) && "expected yield op terminator");
281
282 // Bufferize maskable op. By default, place the buffer allocation right before
283 // the mask op.
284 Value alloc = bufferizeToAllocation(
285 rewriter, options, op: maskOp.getMaskableOp(), memorySpace,
286 /*insertionPoint=*/insertionPoint ? insertionPoint : maskOp);
287
288 if (options.bufferizeDestinationOnly)
289 return alloc;
290
291 // Bufferize terminator.
292 rewriter.setInsertionPoint(yieldOp);
293 if (failed(Result: cast<bufferization::BufferizableOpInterface>(Val: yieldOp).bufferize(
294 rewriter, options: bufferizationOptions, state&: bufferizationState)))
295 return nullptr;
296
297 // Erase dead to_tensor ops inside of the mask op. This is necessary because
298 // there only be one op (apart from the terminator) inside the mask op.
299 // TODO: Remove dead to_tensor ops more aggressively during bufferization.
300 SmallVector<Operation *> toTensorOps;
301 maskOp.walk(callback: [&](bufferization::ToTensorOp toTensorOp) {
302 if (toTensorOp->getUses().empty())
303 toTensorOps.push_back(Elt: toTensorOp.getOperation());
304 });
305 for (Operation *op : toTensorOps)
306 rewriter.eraseOp(op);
307
308 // Bufferize mask op.
309 SmallVector<OpOperand *> resultUses;
310 for (Value result : maskOp.getResults())
311 if (isa<TensorType>(Val: result.getType()))
312 for (OpOperand &use : result.getUses())
313 resultUses.push_back(Elt: &use);
314 rewriter.setInsertionPoint(maskOp);
315 if (failed(
316 Result: cast<bufferization::BufferizableOpInterface>(Val: maskOp.getOperation())
317 .bufferize(rewriter, options: bufferizationOptions, state&: bufferizationState)))
318 return nullptr;
319
320 // Set "restrict" attribute, indicating that no other tensor aliases with
321 // this tensor. That is because we just allocated a new buffer for the tensor.
322 for (OpOperand *resultUse : resultUses) {
323 auto toTensorOp =
324 resultUse->get().getDefiningOp<bufferization::ToTensorOp>();
325 assert(toTensorOp && "expected to_tensor op");
326 rewriter.modifyOpInPlace(root: toTensorOp, callable: [&]() {
327 toTensorOp.setRestrict(true);
328 toTensorOp.setWritable(true);
329 });
330 }
331
332 return alloc;
333}
334
335Value linalg::bufferizeToAllocation(
336 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
337 bufferization::AllocTensorOp allocTensorOp, Attribute memorySpace,
338 Operation *insertionPoint) {
339 Location loc = allocTensorOp.getLoc();
340 OpBuilder::InsertionGuard g(rewriter);
341 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : allocTensorOp);
342 bufferization::BufferizationOptions bufferizationOptions;
343
344 // Create buffer allocation.
345 Value alloc = createAllocationForTensor(
346 rewriter, loc, value: allocTensorOp.getResult(), options, memorySpace);
347
348 // Create bufferization.to_tensor with "restrict" and "writable". The returned
349 // tensor is a new buffer allocation, so it does not alias with any buffer.
350 Value toTensorOp = bufferization::ToTensorOp::create(
351 builder&: rewriter, location: loc, result: allocTensorOp.getResult().getType(), buffer: alloc,
352 /*restrict=*/true,
353 /*writable=*/true);
354 rewriter.replaceOp(op: allocTensorOp, newValues: toTensorOp);
355 return alloc;
356}
357
358/// Lower tensor.from_elements to a sequence of chained tensor.insert.
359FailureOr<Operation *> mlir::linalg::rewriteInDestinationPassingStyle(
360 RewriterBase &rewriter, tensor::FromElementsOp fromElementsOp) {
361 Location loc = fromElementsOp.getLoc();
362 RankedTensorType tensorType =
363 cast<RankedTensorType>(Val: fromElementsOp.getType());
364 auto shape = tensorType.getShape();
365
366 // Create tensor.empty.
367 auto emptyOp = EmptyOp::create(builder&: rewriter, location: loc, result: tensorType, dynamicSizes: ValueRange());
368
369 // Case: tensor<elem_type>.
370 if (shape.empty()) {
371 Operation *res = rewriter.replaceOpWithNewOp<tensor::InsertOp>(
372 op: fromElementsOp, args: fromElementsOp.getElements().front(),
373 args: emptyOp.getResult(), args: ValueRange());
374 return res;
375 }
376
377 // Create constants for the range of possible indices [0, max{shape_i}).
378 auto maxDim = *llvm::max_element(Range&: shape);
379 SmallVector<Value, 2> constants;
380 constants.reserve(N: maxDim);
381 for (int i = 0; i < maxDim; ++i)
382 constants.push_back(Elt: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i));
383
384 // Traverse all elements and create tensor.insert ops.
385 auto elementIt = fromElementsOp.getElements().begin();
386 SmallVector<Value, 2> indices(tensorType.getRank(), constants[0]);
387 Value result = createInserts(rewriter, loc, /*dim=*/0, destination: emptyOp.getResult(),
388 shape, constants, elementIt, indices);
389
390 // Replace tensor.from_elements.
391 rewriter.replaceOp(op: fromElementsOp, newValues: result);
392 return result.getDefiningOp();
393}
394
395/// Lower tensor.generate to linalg.generic.
396FailureOr<Operation *>
397mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
398 tensor::GenerateOp generateOp) {
399 // Only ops with exactly one block are supported.
400 if (!generateOp.getBody().hasOneBlock())
401 return failure();
402
403 Location loc = generateOp.getLoc();
404 RankedTensorType tensorType = cast<RankedTensorType>(Val: generateOp.getType());
405
406 // Create tensor.empty.
407 auto emptyOp = EmptyOp::create(builder&: rewriter, location: loc, result: tensorType,
408 dynamicSizes: generateOp.getDynamicExtents());
409
410 // Create linalg.generic.
411 SmallVector<utils::IteratorType> iteratorTypes(tensorType.getRank(),
412 utils::IteratorType::parallel);
413 SmallVector<AffineMap> indexingMaps(
414 1, rewriter.getMultiDimIdentityMap(rank: tensorType.getRank()));
415 auto genericOp = linalg::GenericOp::create(
416 builder&: rewriter, location: loc, resultTensorTypes: tensorType, /*inputs=*/ValueRange(),
417 /*outputs=*/ValueRange{emptyOp.getResult()}, /*indexingMaps=*/
418 indexingMaps, iteratorTypes);
419 Block *body = rewriter.createBlock(parent: &genericOp->getRegion(index: 0), insertPt: {},
420 argTypes: tensorType.getElementType(), locs: loc);
421 rewriter.setInsertionPointToStart(body);
422 SmallVector<Value> bbArgReplacements;
423 for (int64_t i = 0; i < tensorType.getRank(); ++i)
424 bbArgReplacements.push_back(Elt: linalg::IndexOp::create(builder&: rewriter, location: loc, dim: i));
425 rewriter.mergeBlocks(source: &generateOp.getBody().front(), dest: body, argValues: bbArgReplacements);
426
427 // Update terminator.
428 auto yieldOp = cast<tensor::YieldOp>(Val: body->getTerminator());
429 rewriter.replaceOpWithNewOp<linalg::YieldOp>(op: yieldOp, args: yieldOp.getValue());
430
431 // Replace tensor.generate.
432 rewriter.replaceOp(op: generateOp, newValues: genericOp->getResult(idx: 0));
433 return genericOp.getOperation();
434}
435
436/// Lower tensor.pad to linalg.generic + tensor.insert_slice.
437FailureOr<Operation *>
438mlir::linalg::rewriteInDestinationPassingStyle(RewriterBase &rewriter,
439 tensor::PadOp padOp) {
440 // Only ops with exactly one block are supported.
441 if (!padOp.getBodyRegion().hasOneBlock())
442 return failure();
443
444 // Create tensor.empty.
445 Location loc = padOp.getLoc();
446 RankedTensorType resultType = padOp.getResultType();
447 ReifiedRankedShapedTypeDims reifiedShape;
448 if (failed(Result: reifyResultShapes(b&: rewriter, op: padOp, reifiedReturnShapes&: reifiedShape)))
449 return rewriter.notifyMatchFailure(
450 arg&: padOp, msg: "failed to reify tensor.pad op result shape");
451 SmallVector<Value> dynamicSizes;
452 for (int64_t i = 0; i < resultType.getRank(); ++i)
453 if (resultType.isDynamicDim(idx: i))
454 dynamicSizes.push_back(Elt: cast<Value>(Val&: reifiedShape[0][i]));
455
456 // If the `padOp` has a nofold attribute and all paddings are known to be 0,
457 // explicitly insert a `linalg.copy`.
458 if (padOp.getNofoldAttr() &&
459 llvm::all_of(Range: padOp.getMixedLowPad(), P: isZeroInteger) &&
460 llvm::all_of(Range: padOp.getMixedHighPad(), P: isZeroInteger)) {
461 using bufferization::AllocTensorOp;
462 Value allocated =
463 AllocTensorOp::create(builder&: rewriter, location: loc, type: resultType, dynamicSizes);
464 auto copyOp = rewriter.replaceOpWithNewOp<linalg::CopyOp>(
465 op: padOp, args: padOp.getSource(), args&: allocated);
466 return copyOp.getOperation();
467 }
468
469 Value empty = EmptyOp::create(builder&: rewriter, location: loc, result: resultType, dynamicSizes);
470 // Create linalg.fill or linalg.generic.
471 Operation *fillOp = movePaddingToFillOrGenericOp(rewriter, loc, padOp, dest: empty);
472 rewriter.setInsertionPointAfter(fillOp);
473
474 // Create tensor::InsertSliceOp.
475 SmallVector<OpFoldResult> sliceSizes =
476 getMixedSizes(builder&: rewriter, loc, value: padOp.getSource());
477 SmallVector<OpFoldResult> sliceStrides(resultType.getRank(),
478 rewriter.getIndexAttr(value: 1));
479 auto insertSliceOp = rewriter.replaceOpWithNewOp<tensor::InsertSliceOp>(
480 op: padOp, args: padOp.getSource(), args: fillOp->getResult(idx: 0),
481 /*offsets=*/args: padOp.getMixedLowPad(), args&: sliceSizes, args&: sliceStrides);
482 return insertSliceOp.getOperation();
483}
484
485Value linalg::bufferizeToAllocation(
486 RewriterBase &rewriter, const linalg::BufferizeToAllocationOptions &options,
487 Operation *op, Attribute memorySpace, Operation *insertionPoint) {
488 using namespace bufferization;
489
490 // Call specialized overload for certain ops.
491 if (auto padOp = dyn_cast<tensor::PadOp>(Val: op))
492 return bufferizeToAllocation(rewriter, options, padOp, memorySpace);
493 if (auto maskOp = dyn_cast<vector::MaskOp>(Val: op))
494 return bufferizeToAllocation(rewriter, options, maskOp, memorySpace);
495 if (auto allocTensorOp = dyn_cast<bufferization::AllocTensorOp>(Val: op))
496 return bufferizeToAllocation(rewriter, options, allocTensorOp, memorySpace);
497
498 // Only bufferizable ops are supported.
499 auto bufferizableOp = dyn_cast<BufferizableOpInterface>(Val: op);
500 if (!bufferizableOp)
501 return nullptr;
502
503 // Should the bufferization options and states be function arguments?
504 BufferizationOptions bufferizationOptions;
505 AnalysisState analysisState(bufferizationOptions);
506 BufferizationState bufferizationState;
507
508#ifndef NDEBUG
509 if (!options.bufferizeDestinationOnly) {
510 // Ops with nested tensor ops are not supported yet. At the moment, this
511 // function just bufferizes the given op itself, but not its body.
512 op->walk([&](Operation *nestedOp) {
513 if (op == nestedOp)
514 return;
515 if (llvm::any_of(nestedOp->getOperands(),
516 [](Value v) { return isa<TensorType>(v.getType()); }))
517 llvm_unreachable("ops with nested tensor ops are not supported yet");
518 if (llvm::any_of(nestedOp->getResults(),
519 [](Value v) { return isa<TensorType>(v.getType()); }))
520 llvm_unreachable("ops with nested tensor ops are not supported yet");
521 });
522 }
523#endif // NDEBUG
524
525 // Gather tensor results.
526 SmallVector<OpResult> tensorResults;
527 for (OpResult result : op->getResults()) {
528 if (!isa<TensorType>(Val: result.getType()))
529 continue;
530 // Unranked tensors are not supported
531 if (!isa<RankedTensorType>(Val: result.getType()))
532 return nullptr;
533 // Ops that bufferize to an allocation are not supported.
534 if (bufferizableOp.bufferizesToAllocation(value: result))
535 return nullptr;
536 tensorResults.push_back(Elt: result);
537 }
538
539 // Gather all operands that should bufferize to a new allocation. I.e.,
540 // bufferize out-of-place.
541 SmallVector<OpOperand *> outOfPlaceOperands, resultUses;
542 auto addOutOfPlaceOperand = [&](OpOperand *operand) {
543 if (!llvm::is_contained(Range&: outOfPlaceOperands, Element: operand))
544 outOfPlaceOperands.push_back(Elt: operand);
545 };
546 for (OpResult result : tensorResults) {
547 AliasingOpOperandList aliasingOperands =
548 analysisState.getAliasingOpOperands(value: result);
549 for (const AliasingOpOperand &operand : aliasingOperands) {
550 addOutOfPlaceOperand(operand.opOperand);
551 for (OpOperand &resultUse : result.getUses())
552 resultUses.push_back(Elt: &resultUse);
553 }
554 }
555 for (OpOperand &operand : op->getOpOperands()) {
556 if (!analysisState.bufferizesToMemoryWrite(opOperand&: operand))
557 continue;
558 if (!isa<RankedTensorType>(Val: operand.get().getType()))
559 continue;
560 addOutOfPlaceOperand(&operand);
561 }
562 // TODO: Support multiple buffers.
563 if (outOfPlaceOperands.size() != 1)
564 return nullptr;
565
566 // Allocate buffers.
567 OpBuilder::InsertionGuard g(rewriter);
568 rewriter.setInsertionPoint(insertionPoint ? insertionPoint : op);
569 SmallVector<Value> allocs;
570 for (OpOperand *operand : outOfPlaceOperands) {
571 Value alloc = createAllocationForTensor(
572 rewriter, loc: op->getLoc(), value: operand->get(), options, memorySpace);
573 allocs.push_back(Elt: alloc);
574 if (!analysisState.findDefinitions(opOperand: operand).empty()) {
575 // Initialize buffer with a copy of the operand data. Not needed if the
576 // tensor is uninitialized.
577 createMemcpy(b&: rewriter, loc: op->getLoc(), tensorSource: operand->get(), memrefDest: alloc, options);
578 }
579 rewriter.modifyOpInPlace(root: op, callable: [&]() {
580 auto toTensorOp = ToTensorOp::create(builder&: rewriter, location: op->getLoc(),
581 result: operand->get().getType(), buffer: alloc);
582 operand->set(toTensorOp);
583 if (options.bufferizeDestinationOnly) {
584 rewriter.modifyOpInPlace(root: toTensorOp, callable: [&]() {
585 toTensorOp.setRestrict(true);
586 toTensorOp.setWritable(true);
587 });
588 }
589 });
590 }
591
592 if (options.bufferizeDestinationOnly)
593 return allocs.front();
594
595 // Bufferize the op.
596 rewriter.setInsertionPoint(op);
597 if (failed(Result: bufferizableOp.bufferize(rewriter, options: bufferizationOptions,
598 state&: bufferizationState)))
599 return nullptr;
600
601 // Set "restrict" attribute, indicating that no other tensor aliases with
602 // this tensor. That is because we just allocated a new buffer for the tensor.
603 for (OpOperand *resultUse : resultUses) {
604 auto toTensorOp = resultUse->get().getDefiningOp<ToTensorOp>();
605 assert(toTensorOp && "expected to_tensor op");
606 rewriter.modifyOpInPlace(root: toTensorOp, callable: [&]() {
607 toTensorOp.setRestrict(true);
608 toTensorOp.setWritable(true);
609 });
610 }
611 return allocs.front();
612}
613
614namespace {
615
616template <typename OpTy>
617LogicalResult rewriteOpInDestinationPassingStyle(OpTy op,
618 PatternRewriter &rewriter) {
619 return linalg::rewriteInDestinationPassingStyle(rewriter, op);
620}
621
622} // namespace
623
624void linalg::populateConvertToDestinationStylePatterns(
625 RewritePatternSet &patterns) {
626 patterns.add(implFn: rewriteOpInDestinationPassingStyle<tensor::FromElementsOp>);
627 patterns.add(implFn: rewriteOpInDestinationPassingStyle<tensor::GenerateOp>);
628 patterns.add(implFn: rewriteOpInDestinationPassingStyle<tensor::PadOp>);
629}
630