1//===- Shape.cpp - MLIR Shape Operations ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Shape/IR/Shape.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
13#include "mlir/Dialect/CommonFolders.h"
14#include "mlir/Dialect/Tensor/IR/Tensor.h"
15#include "mlir/Dialect/Traits.h"
16#include "mlir/Dialect/UB/IR/UBOps.h"
17#include "mlir/IR/Builders.h"
18#include "mlir/IR/BuiltinTypes.h"
19#include "mlir/IR/Matchers.h"
20#include "mlir/IR/PatternMatch.h"
21#include "mlir/IR/TypeUtilities.h"
22#include "mlir/Interfaces/FunctionImplementation.h"
23#include "mlir/Transforms/InliningUtils.h"
24#include "llvm/ADT/SetOperations.h"
25#include "llvm/ADT/SmallVectorExtras.h"
26#include "llvm/Support/raw_ostream.h"
27#include <utility>
28
29using namespace mlir;
30using namespace mlir::shape;
31
32#include "mlir/Dialect/Shape/IR/ShapeOpsDialect.cpp.inc"
33
34namespace {
35#include "ShapeCanonicalization.inc"
36} // namespace
37
38RankedTensorType shape::getExtentTensorType(MLIRContext *ctx, int64_t rank) {
39 return RankedTensorType::get(shape: {rank}, elementType: IndexType::get(context: ctx));
40}
41
42bool shape::isExtentTensorType(Type type) {
43 auto ranked = llvm::dyn_cast<RankedTensorType>(Val&: type);
44 return ranked && ranked.getRank() == 1 && ranked.getElementType().isIndex();
45}
46
47LogicalResult shape::getShapeVec(Value input,
48 SmallVectorImpl<int64_t> &shapeValues) {
49 if (auto inputOp = input.getDefiningOp<ShapeOfOp>()) {
50 auto type = llvm::cast<ShapedType>(Val: inputOp.getArg().getType());
51 if (!type.hasRank())
52 return failure();
53 llvm::append_range(C&: shapeValues, R: type.getShape());
54 return success();
55 }
56 DenseIntElementsAttr attr;
57 if (matchPattern(value: input, pattern: m_Constant(bind_value: &attr))) {
58 llvm::append_range(C&: shapeValues, R: attr.getValues<int64_t>());
59 return success();
60 }
61 return failure();
62}
63
64static bool isErrorPropagationPossible(TypeRange operandTypes) {
65 return llvm::any_of(Range&: operandTypes,
66 P: llvm::IsaPred<SizeType, ShapeType, ValueShapeType>);
67}
68
69static LogicalResult verifySizeOrIndexOp(Operation *op) {
70 assert(op != nullptr && op->getNumResults() == 1);
71 Type resultTy = op->getResultTypes().front();
72 if (isErrorPropagationPossible(operandTypes: op->getOperandTypes())) {
73 if (!llvm::isa<SizeType>(Val: resultTy))
74 return op->emitOpError()
75 << "if at least one of the operands can hold error values then "
76 "the result must be of type `size` to propagate them";
77 }
78 return success();
79}
80
81static LogicalResult verifyShapeOrExtentTensorOp(Operation *op) {
82 assert(op != nullptr && op->getNumResults() == 1);
83 Type resultTy = op->getResultTypes().front();
84 if (isErrorPropagationPossible(operandTypes: op->getOperandTypes())) {
85 if (!llvm::isa<ShapeType>(Val: resultTy))
86 return op->emitOpError()
87 << "if at least one of the operands can hold error values then "
88 "the result must be of type `shape` to propagate them";
89 }
90 return success();
91}
92
93template <typename... Ty>
94static bool eachHasOnlyOneOfTypes(TypeRange typeRange) {
95 return typeRange.size() == 1 && llvm::isa<Ty...>(typeRange.front());
96}
97
98template <typename... Ty, typename... ranges>
99static bool eachHasOnlyOneOfTypes(TypeRange l, ranges... rs) {
100 return eachHasOnlyOneOfTypes<Ty...>(l) && eachHasOnlyOneOfTypes<Ty...>(rs...);
101}
102
103//===----------------------------------------------------------------------===//
104// InlinerInterface
105//===----------------------------------------------------------------------===//
106
107namespace {
108/// This class defines the interface for inlining shape dialect ops.
109struct ShapeInlinerInterface : public DialectInlinerInterface {
110 using DialectInlinerInterface::DialectInlinerInterface;
111
112 // Returns true if the given region 'src' can be inlined into the region
113 // 'dest' that is attached to an operation registered to the current dialect.
114 bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
115 IRMapping &) const final {
116 return true;
117 }
118
119 // Returns true if the given operation 'op', that is registered to this
120 // dialect, can be inlined into the region 'dest' that is attached to an
121 // operation registered to the current dialect.
122 bool isLegalToInline(Operation *op, Region *dest, bool wouldBeCloned,
123 IRMapping &) const final {
124 return true;
125 }
126};
127} // namespace
128
129void ShapeDialect::initialize() {
130 addOperations<
131#define GET_OP_LIST
132#include "mlir/Dialect/Shape/IR/ShapeOps.cpp.inc"
133 >();
134 addTypes<
135#define GET_TYPEDEF_LIST
136#include "mlir/Dialect/Shape/IR/ShapeOpsTypes.cpp.inc"
137 >();
138 addInterfaces<ShapeInlinerInterface>();
139 // Allow unknown operations during prototyping and testing. As the dialect is
140 // still evolving it makes it simple to start with an unregistered ops and
141 // try different variants before actually defining the op.
142 allowUnknownOperations();
143 declarePromisedInterfaces<bufferization::BufferizableOpInterface, AssumingOp,
144 AssumingYieldOp>();
145}
146
147Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
148 Attribute value, Type type,
149 Location loc) {
150 if (auto poison = dyn_cast<ub::PoisonAttr>(Val&: value))
151 return ub::PoisonOp::create(builder, location: loc, result: type, value: poison);
152
153 if (llvm::isa<ShapeType>(Val: type) || isExtentTensorType(type))
154 return ConstShapeOp::create(builder, location: loc, result: type,
155 shape: llvm::cast<DenseIntElementsAttr>(Val&: value));
156 if (llvm::isa<SizeType>(Val: type))
157 return ConstSizeOp::create(builder, location: loc, result: type,
158 value: llvm::cast<IntegerAttr>(Val&: value));
159 if (llvm::isa<WitnessType>(Val: type))
160 return ConstWitnessOp::create(builder, location: loc, result: type,
161 passing: llvm::cast<BoolAttr>(Val&: value));
162
163 return arith::ConstantOp::materialize(builder, value, type, loc);
164}
165
166LogicalResult ShapeDialect::verifyOperationAttribute(Operation *op,
167 NamedAttribute attribute) {
168 // Verify shape.lib attribute.
169 if (attribute.getName() == "shape.lib") {
170 if (!op->hasTrait<OpTrait::SymbolTable>())
171 return op->emitError(
172 message: "shape.lib attribute may only be on op implementing SymbolTable");
173
174 if (auto symbolRef = llvm::dyn_cast<SymbolRefAttr>(Val: attribute.getValue())) {
175 auto *symbol = SymbolTable::lookupSymbolIn(op, symbol: symbolRef);
176 if (!symbol)
177 return op->emitError(message: "shape function library ")
178 << symbolRef << " not found";
179 return isa<shape::FunctionLibraryOp>(Val: symbol)
180 ? success()
181 : op->emitError()
182 << symbolRef << " required to be shape function library";
183 }
184
185 if (auto arr = llvm::dyn_cast<ArrayAttr>(Val: attribute.getValue())) {
186 // Verify all entries are function libraries and mappings in libraries
187 // refer to unique ops.
188 DenseSet<StringAttr> key;
189 for (auto it : arr) {
190 if (!llvm::isa<SymbolRefAttr>(Val: it))
191 return op->emitError(
192 message: "only SymbolRefAttr allowed in shape.lib attribute array");
193
194 auto shapeFnLib = dyn_cast_or_null<shape::FunctionLibraryOp>(
195 Val: SymbolTable::lookupSymbolIn(op, symbol: llvm::cast<SymbolRefAttr>(Val&: it)));
196 if (!shapeFnLib)
197 return op->emitError()
198 << it << " does not refer to FunctionLibraryOp";
199 for (auto mapping : shapeFnLib.getMapping()) {
200 if (!key.insert(V: mapping.getName()).second) {
201 return op->emitError(message: "only one op to shape mapping allowed, found "
202 "multiple for `")
203 << mapping.getName() << "`";
204 }
205 }
206 }
207 return success();
208 }
209
210 return op->emitError(message: "only SymbolRefAttr or array of SymbolRefAttrs "
211 "allowed as shape.lib attribute");
212 }
213 return success();
214}
215
216//===----------------------------------------------------------------------===//
217// AnyOp
218//===----------------------------------------------------------------------===//
219
220// TODO: Canonicalization should be implemented for shapes that can be
221// determined through mixtures of the known dimensions of the inputs.
222OpFoldResult AnyOp::fold(FoldAdaptor adaptor) {
223 // Only the last operand is checked because AnyOp is commutative.
224 if (adaptor.getInputs().back())
225 return adaptor.getInputs().back();
226
227 return nullptr;
228}
229
230//===----------------------------------------------------------------------===//
231// AssumingOp
232//===----------------------------------------------------------------------===//
233
234ParseResult AssumingOp::parse(OpAsmParser &parser, OperationState &result) {
235 result.regions.reserve(N: 1);
236 Region *doRegion = result.addRegion();
237
238 auto &builder = parser.getBuilder();
239 OpAsmParser::UnresolvedOperand cond;
240 if (parser.parseOperand(result&: cond) ||
241 parser.resolveOperand(operand: cond, type: builder.getType<WitnessType>(),
242 result&: result.operands))
243 return failure();
244
245 // Parse optional results type list.
246 if (parser.parseOptionalArrowTypeList(result&: result.types))
247 return failure();
248
249 // Parse the region and add a terminator if elided.
250 if (parser.parseRegion(region&: *doRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {}))
251 return failure();
252 AssumingOp::ensureTerminator(region&: *doRegion, builder&: parser.getBuilder(), loc: result.location);
253
254 // Parse the optional attribute list.
255 if (parser.parseOptionalAttrDict(result&: result.attributes))
256 return failure();
257 return success();
258}
259
260void AssumingOp::print(OpAsmPrinter &p) {
261 bool yieldsResults = !getResults().empty();
262
263 p << " " << getWitness();
264 if (yieldsResults)
265 p << " -> (" << getResultTypes() << ")";
266 p << ' ';
267 p.printRegion(blocks&: getDoRegion(),
268 /*printEntryBlockArgs=*/false,
269 /*printBlockTerminators=*/yieldsResults);
270 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
271}
272
273namespace {
274// Removes AssumingOp with a passing witness and inlines the region.
275struct AssumingWithTrue : public OpRewritePattern<AssumingOp> {
276 using OpRewritePattern<AssumingOp>::OpRewritePattern;
277
278 LogicalResult matchAndRewrite(AssumingOp op,
279 PatternRewriter &rewriter) const override {
280 auto witness = op.getWitness().getDefiningOp<ConstWitnessOp>();
281 if (!witness || !witness.getPassingAttr())
282 return failure();
283
284 AssumingOp::inlineRegionIntoParent(op, rewriter);
285 return success();
286 }
287};
288
289struct AssumingOpRemoveUnusedResults : public OpRewritePattern<AssumingOp> {
290 using OpRewritePattern<AssumingOp>::OpRewritePattern;
291
292 LogicalResult matchAndRewrite(AssumingOp op,
293 PatternRewriter &rewriter) const override {
294 Block *body = op.getBody();
295 auto yieldOp = llvm::cast<AssumingYieldOp>(Val: body->getTerminator());
296
297 // Find used values.
298 SmallVector<Value, 4> newYieldOperands;
299 for (auto [opResult, yieldOperand] :
300 llvm::zip(t: op.getResults(), u: yieldOp.getOperands())) {
301 if (!opResult.getUses().empty()) {
302 newYieldOperands.push_back(Elt: yieldOperand);
303 }
304 }
305
306 // Rewrite only if redundant results exist.
307 if (newYieldOperands.size() == yieldOp->getNumOperands())
308 return failure();
309
310 // Replace yield op in the old assuming op's body and move the entire region
311 // to the new assuming op.
312 rewriter.setInsertionPointToEnd(body);
313 auto newYieldOp =
314 rewriter.replaceOpWithNewOp<AssumingYieldOp>(op: yieldOp, args&: newYieldOperands);
315 rewriter.setInsertionPoint(op);
316 auto newOp = AssumingOp::create(
317 builder&: rewriter, location: op.getLoc(), results: newYieldOp->getOperandTypes(), witness: op.getWitness());
318 newOp.getDoRegion().takeBody(other&: op.getDoRegion());
319
320 // Use the new results to replace the previously used ones.
321 SmallVector<Value, 4> replacementValues;
322 auto src = newOp.getResults().begin();
323 for (auto it : op.getResults()) {
324 if (it.getUses().empty())
325 replacementValues.push_back(Elt: nullptr);
326 else
327 replacementValues.push_back(Elt: *src++);
328 }
329 rewriter.replaceOp(op, newValues: replacementValues);
330 return success();
331 }
332};
333} // namespace
334
335void AssumingOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
336 MLIRContext *context) {
337 patterns.add<AssumingOpRemoveUnusedResults, AssumingWithTrue>(arg&: context);
338}
339
340// See RegionBranchOpInterface in Interfaces/ControlFlowInterfaces.td
341void AssumingOp::getSuccessorRegions(
342 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
343 // AssumingOp has unconditional control flow into the region and back to the
344 // parent, so return the correct RegionSuccessor purely based on the index
345 // being None or 0.
346 if (!point.isParent()) {
347 regions.push_back(Elt: RegionSuccessor(getOperation()));
348 return;
349 }
350
351 regions.push_back(Elt: RegionSuccessor(&getDoRegion()));
352}
353
354ValueRange AssumingOp::getSuccessorInputs(RegionSuccessor successor) {
355 return successor.isOperation() ? ValueRange(getResults()) : ValueRange();
356}
357
358void AssumingOp::inlineRegionIntoParent(AssumingOp &op,
359 PatternRewriter &rewriter) {
360 auto *blockBeforeAssuming = rewriter.getInsertionBlock();
361 auto *assumingBlock = op.getBody();
362 auto initPosition = rewriter.getInsertionPoint();
363 auto *blockAfterAssuming =
364 rewriter.splitBlock(block: blockBeforeAssuming, before: initPosition);
365
366 // Remove the AssumingOp and AssumingYieldOp.
367 auto &yieldOp = assumingBlock->back();
368 rewriter.inlineRegionBefore(region&: op.getDoRegion(), before: blockAfterAssuming);
369 rewriter.replaceOp(op, newValues: yieldOp.getOperands());
370 rewriter.eraseOp(op: &yieldOp);
371
372 // Merge blocks together as there was no branching behavior from the
373 // AssumingOp.
374 rewriter.mergeBlocks(source: assumingBlock, dest: blockBeforeAssuming);
375 rewriter.mergeBlocks(source: blockAfterAssuming, dest: blockBeforeAssuming);
376}
377
378void AssumingOp::build(
379 OpBuilder &builder, OperationState &result, Value witness,
380 function_ref<SmallVector<Value, 2>(OpBuilder &, Location)> bodyBuilder) {
381 OpBuilder::InsertionGuard g(builder);
382
383 result.addOperands(newOperands: witness);
384 Region *bodyRegion = result.addRegion();
385 builder.createBlock(parent: bodyRegion);
386
387 // Build body.
388 SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
389 AssumingYieldOp::create(builder, location: result.location, operands: yieldValues);
390
391 SmallVector<Type, 2> assumingTypes;
392 for (Value v : yieldValues)
393 assumingTypes.push_back(Elt: v.getType());
394 result.addTypes(newTypes: assumingTypes);
395}
396
397//===----------------------------------------------------------------------===//
398// AddOp
399//===----------------------------------------------------------------------===//
400
401LogicalResult mlir::shape::AddOp::inferReturnTypes(
402 MLIRContext *context, std::optional<Location> location,
403 AddOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
404 if (llvm::isa<SizeType>(Val: adaptor.getLhs().getType()) ||
405 llvm::isa<SizeType>(Val: adaptor.getRhs().getType()))
406 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
407 else
408 inferredReturnTypes.assign(IL: {IndexType::get(context)});
409 return success();
410}
411
412bool mlir::shape::AddOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
413 // SizeType is compatible with IndexType.
414 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
415}
416
417OpFoldResult mlir::shape::AddOp::fold(FoldAdaptor adaptor) {
418 // add(x, 0) -> x
419 if (matchPattern(value: getRhs(), pattern: m_Zero()))
420 return getLhs();
421
422 return constFoldBinaryOp<IntegerAttr>(
423 operands: adaptor.getOperands(),
424 calculate: [](APInt a, const APInt &b) { return std::move(a) + b; });
425}
426
427LogicalResult shape::AddOp::verify() { return verifySizeOrIndexOp(op: *this); }
428
429//===----------------------------------------------------------------------===//
430// AssumingAllOp
431//===----------------------------------------------------------------------===//
432
433namespace {
434
435// Merge multiple `shape.assuming_all` operations together.
436//
437// %0 = shape.assuming_all %w0, %w1
438// %1 = shape.assuming_all %w2, %0
439//
440// to:
441//
442// %0 = shape.assuming_all %w0, %w2, %w2
443struct MergeAssumingAllOps : public OpRewritePattern<AssumingAllOp> {
444 using OpRewritePattern<AssumingAllOp>::OpRewritePattern;
445
446 LogicalResult matchAndRewrite(AssumingAllOp op,
447 PatternRewriter &rewriter) const override {
448 SmallVector<Value> operands;
449
450 for (Value operand : op.getInputs()) {
451 if (auto assumeAll = operand.getDefiningOp<AssumingAllOp>())
452 operands.append(in_start: assumeAll.operand_begin(), in_end: assumeAll->operand_end());
453 else
454 operands.push_back(Elt: operand);
455 }
456
457 // We didn't find any other `assuming_all` ops to merge with.
458 if (operands.size() == op.getNumOperands())
459 return failure();
460
461 // Replace with a new `assuming_all` operation with merged constraints.
462 rewriter.replaceOpWithNewOp<AssumingAllOp>(op, args&: operands);
463 return success();
464 }
465};
466
467// Eliminate `cstr_broadcastable` operands from `assuming_all` operation that
468// are subsumed by others.
469//
470// %0 = shape.cstr_broadcastable %shape0, %shape1
471// %1 = shape.cstr_broadcastable %shape0, %shape1, %shape2
472//
473// %2 = shape.cstr_broadcastable %shape3, %shape4
474// %3 = shape.cstr_broadcastable %shape3, %shape4, %shape5
475//
476// %4 = shape.assuming_all %0, %1, %2, %3
477//
478// to:
479//
480// %0 = shape.cstr_broadcastable %shape0, %shape1, %shape2
481// %1 = shape.cstr_broadcastable %shape3, %shape4, %shape5
482// %2 = shape.assuming_all %0, %1
483//
484// In this example if shapes [0, 1, 2] are broadcastable, then it means that
485// shapes [0, 1] are broadcastable too, and can be removed from the list of
486// constraints. If shapes [0, 1, 2] are not broadcastable, then it doesn't
487// matter if shapes [0, 1] are broadcastable (same for shapes [3, 4, 5]).
488struct AssumingAllOfCstrBroadcastable : public OpRewritePattern<AssumingAllOp> {
489 using OpRewritePattern<AssumingAllOp>::OpRewritePattern;
490
491 LogicalResult matchAndRewrite(AssumingAllOp op,
492 PatternRewriter &rewriter) const override {
493 // Collect all `CstrBroadcastableOp` operands first.
494 SetVector<CstrBroadcastableOp> operands;
495 for (Value operand : op.getInputs()) {
496 // TODO: Apply this optimization if some of the witnesses are not
497 // produced by the `cstr_broadcastable`.
498 auto broadcastable = operand.getDefiningOp<CstrBroadcastableOp>();
499 if (!broadcastable)
500 return failure();
501
502 operands.insert(X: broadcastable);
503 }
504
505 // Skip trivial `assuming_all` operations.
506 if (operands.size() <= 1)
507 return failure();
508
509 // Collect shapes checked by `cstr_broadcastable` operands.
510 SmallVector<std::pair<CstrBroadcastableOp, DenseSet<Value>>> shapes;
511 for (auto cstr : operands) {
512 DenseSet<Value> shapesSet(cstr->operand_begin(), cstr->operand_end());
513 shapes.emplace_back(Args&: cstr, Args: std::move(shapesSet));
514 }
515
516 // Sort by the number of shape operands (larger to smaller).
517 llvm::sort(C&: shapes, Comp: [](auto a, auto b) {
518 return a.first.getNumOperands() > b.first.getNumOperands();
519 });
520
521 // We start from the `cst_broadcastable` operations with largest number of
522 // shape operands, and remove redundant `cst_broadcastable` operations. We
523 // do this until we find a set of `cst_broadcastable` operations with
524 // non-overlapping constraints.
525 SmallVector<CstrBroadcastableOp> markedForErase;
526
527 for (unsigned i = 0; i < shapes.size(); ++i) {
528 auto isSubset = [&](auto pair) {
529 return llvm::set_is_subset(pair.second, shapes[i].second);
530 };
531
532 // Keep redundant `cstr_broadcastable` operations to be erased.
533 auto *it = std::remove_if(first: shapes.begin() + i + 1, last: shapes.end(), pred: isSubset);
534 for (auto *it0 = it; it0 < shapes.end(); ++it0)
535 markedForErase.push_back(Elt: it0->first);
536 shapes.erase(CS: it, CE: shapes.end());
537 }
538
539 // We didn't find any operands that could be removed.
540 if (markedForErase.empty())
541 return failure();
542
543 // Collect non-overlapping `cst_broadcastable` constraints.
544 SmallVector<Value> uniqueConstraints;
545 for (auto &shape : shapes)
546 uniqueConstraints.push_back(Elt: shape.first.getResult());
547
548 // Replace with a new `assuming_all` operation ...
549 rewriter.replaceOpWithNewOp<AssumingAllOp>(op, args&: uniqueConstraints);
550
551 // ... and maybe erase `cstr_broadcastable` ops without uses.
552 for (auto &op : markedForErase)
553 if (op->use_empty())
554 rewriter.eraseOp(op);
555
556 return success();
557 }
558};
559
560struct AssumingAllToCstrEqCanonicalization
561 : public OpRewritePattern<AssumingAllOp> {
562 using OpRewritePattern<AssumingAllOp>::OpRewritePattern;
563
564 LogicalResult matchAndRewrite(AssumingAllOp op,
565 PatternRewriter &rewriter) const override {
566 SmallVector<Value, 8> shapes;
567 for (Value w : op.getInputs()) {
568 auto cstrEqOp = w.getDefiningOp<CstrEqOp>();
569 if (!cstrEqOp)
570 return failure();
571 bool disjointShapes = llvm::none_of(Range: cstrEqOp.getShapes(), P: [&](Value s) {
572 return llvm::is_contained(Range&: shapes, Element: s);
573 });
574 if (!shapes.empty() && !cstrEqOp.getShapes().empty() && disjointShapes)
575 return failure();
576 shapes.append(in_start: cstrEqOp.getShapes().begin(), in_end: cstrEqOp.getShapes().end());
577 }
578 rewriter.replaceOpWithNewOp<CstrEqOp>(op, args&: shapes);
579 return success();
580 }
581};
582
583template <typename OpTy>
584struct RemoveDuplicateOperandsPattern : public OpRewritePattern<OpTy> {
585 using OpRewritePattern<OpTy>::OpRewritePattern;
586
587 LogicalResult matchAndRewrite(OpTy op,
588 PatternRewriter &rewriter) const override {
589 // Find unique operands.
590 SetVector<Value> unique(op.operand_begin(), op.operand_end());
591
592 // Reduce op to equivalent with unique operands.
593 if (unique.size() < op.getNumOperands()) {
594 rewriter.replaceOpWithNewOp<OpTy>(
595 op, op->getResultTypes(), unique.takeVector(), op.getProperties(),
596 op->getDiscardableAttrDictionary().getValue());
597 return success();
598 }
599
600 return failure();
601 }
602};
603} // namespace
604
605void AssumingAllOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
606 MLIRContext *context) {
607 patterns
608 .add<MergeAssumingAllOps, AssumingAllOneOp,
609 AssumingAllOfCstrBroadcastable, AssumingAllToCstrEqCanonicalization,
610 RemoveDuplicateOperandsPattern<AssumingAllOp>>(arg&: context);
611}
612
613OpFoldResult AssumingAllOp::fold(FoldAdaptor adaptor) {
614 // Iterate in reverse to first handle all constant operands. They are
615 // guaranteed to be the tail of the inputs because this is commutative.
616 for (int idx = adaptor.getInputs().size() - 1; idx >= 0; idx--) {
617 Attribute a = adaptor.getInputs()[idx];
618 // Cannot fold if any inputs are not constant;
619 if (!a)
620 return nullptr;
621
622 // We do not need to keep statically known values after handling them in
623 // this method.
624 getOperation()->eraseOperand(idx);
625
626 // Always false if any input is statically known false
627 if (!llvm::cast<BoolAttr>(Val&: a).getValue())
628 return a;
629 }
630 // If this is reached, all inputs were statically known passing.
631 return BoolAttr::get(context: getContext(), value: true);
632}
633
634LogicalResult AssumingAllOp::verify() {
635 // Ensure that AssumingAllOp contains at least one operand
636 if (getNumOperands() == 0)
637 return emitOpError(message: "no operands specified");
638
639 return success();
640}
641
642//===----------------------------------------------------------------------===//
643// BroadcastOp
644//===----------------------------------------------------------------------===//
645
646OpFoldResult BroadcastOp::fold(FoldAdaptor adaptor) {
647 if (getShapes().size() == 1) {
648 // Otherwise, we need a cast which would be a canonicalization, not folding.
649 if (getShapes().front().getType() != getType())
650 return nullptr;
651 return getShapes().front();
652 }
653
654 auto firstAttr =
655 dyn_cast_or_null<DenseIntElementsAttr>(Val: adaptor.getShapes().front());
656 if (!firstAttr)
657 return nullptr;
658
659 SmallVector<int64_t, 6> resultShape(firstAttr.getValues<int64_t>());
660
661 for (auto next : adaptor.getShapes().drop_front()) {
662 auto nextAttr = dyn_cast_or_null<DenseIntElementsAttr>(Val&: next);
663 if (!nextAttr)
664 return nullptr;
665 auto nextShape = llvm::to_vector<6>(Range: nextAttr.getValues<int64_t>());
666
667 SmallVector<int64_t, 6> tmpShape;
668 // If the shapes are not compatible, we can't fold it.
669 // TODO: Fold to an "error".
670 if (!OpTrait::util::getBroadcastedShape(shape1: resultShape, shape2: nextShape, resultShape&: tmpShape))
671 return nullptr;
672
673 resultShape.clear();
674 std::copy(first: tmpShape.begin(), last: tmpShape.end(),
675 result: std::back_inserter(x&: resultShape));
676 }
677
678 Builder builder(getContext());
679 return builder.getIndexTensorAttr(values: resultShape);
680}
681
682LogicalResult BroadcastOp::verify() {
683 return verifyShapeOrExtentTensorOp(op: *this);
684}
685
686namespace {
687template <typename OpTy>
688struct RemoveEmptyShapeOperandsPattern : public OpRewritePattern<OpTy> {
689 using OpRewritePattern<OpTy>::OpRewritePattern;
690
691 LogicalResult matchAndRewrite(OpTy op,
692 PatternRewriter &rewriter) const override {
693 auto isPotentiallyNonEmptyShape = [](Value shape) {
694 if (auto extentTensorTy =
695 llvm::dyn_cast<RankedTensorType>(Val: shape.getType())) {
696 if (extentTensorTy.getDimSize(idx: 0) == 0)
697 return false;
698 }
699 if (auto constShape = shape.getDefiningOp<ConstShapeOp>()) {
700 if (constShape.getShape().empty())
701 return false;
702 }
703 return true;
704 };
705 auto newOperands = llvm::filter_to_vector<8>(op->getOperands(),
706 isPotentiallyNonEmptyShape);
707
708 // Replace the op with empty shape constant if all operants are reduced to
709 // be empty.
710 if (newOperands.empty()) {
711 rewriter.replaceOpWithNewOp<ConstShapeOp>(
712 op, op->getResultTypes().front(), rewriter.getIndexTensorAttr(values: {}));
713 return success();
714 }
715
716 // Reduce op to equivalent without empty shape operands.
717 if (newOperands.size() < op.getNumOperands()) {
718 rewriter.replaceOpWithNewOp<OpTy>(
719 op, op->getResultTypes(), newOperands, op.getProperties(),
720 op->getDiscardableAttrDictionary().getValue());
721 return success();
722 }
723
724 return failure();
725 }
726};
727
728struct BroadcastForwardSingleOperandPattern
729 : public OpRewritePattern<BroadcastOp> {
730 using OpRewritePattern<BroadcastOp>::OpRewritePattern;
731
732 LogicalResult matchAndRewrite(BroadcastOp op,
733 PatternRewriter &rewriter) const override {
734 if (op.getNumOperands() != 1)
735 return failure();
736 Value replacement = op.getShapes().front();
737
738 // Insert cast if needed.
739 if (replacement.getType() != op.getType()) {
740 auto loc = op.getLoc();
741 if (llvm::isa<ShapeType>(Val: op.getType())) {
742 replacement = FromExtentTensorOp::create(builder&: rewriter, location: loc, input: replacement);
743 } else {
744 assert(!llvm::isa<ShapeType>(op.getType()) &&
745 !llvm::isa<ShapeType>(replacement.getType()) &&
746 "expect extent tensor cast");
747 replacement =
748 tensor::CastOp::create(builder&: rewriter, location: loc, dest: op.getType(), source: replacement);
749 }
750 }
751
752 rewriter.replaceOp(op, newValues: replacement);
753 return success();
754 }
755};
756
757struct BroadcastFoldConstantOperandsPattern
758 : public OpRewritePattern<BroadcastOp> {
759 using OpRewritePattern<BroadcastOp>::OpRewritePattern;
760
761 LogicalResult matchAndRewrite(BroadcastOp op,
762 PatternRewriter &rewriter) const override {
763 SmallVector<int64_t, 8> foldedConstantShape;
764 SmallVector<Value, 8> newShapeOperands;
765 for (Value shape : op.getShapes()) {
766 if (auto constShape = shape.getDefiningOp<ConstShapeOp>()) {
767 SmallVector<int64_t, 8> newFoldedConstantShape;
768 if (OpTrait::util::getBroadcastedShape(
769 shape1: foldedConstantShape,
770 shape2: llvm::to_vector<8>(Range: constShape.getShape().getValues<int64_t>()),
771 resultShape&: newFoldedConstantShape)) {
772 foldedConstantShape = newFoldedConstantShape;
773 continue;
774 }
775 }
776 newShapeOperands.push_back(Elt: shape);
777 }
778
779 // Need at least two constant operands to fold anything.
780 if (op.getNumOperands() - newShapeOperands.size() < 2)
781 return failure();
782
783 auto foldedConstantOperandsTy = RankedTensorType::get(
784 shape: {static_cast<int64_t>(foldedConstantShape.size())},
785 elementType: rewriter.getIndexType());
786 newShapeOperands.push_back(
787 Elt: ConstShapeOp::create(builder&: rewriter, location: op.getLoc(), result: foldedConstantOperandsTy,
788 shape: rewriter.getIndexTensorAttr(values: foldedConstantShape)));
789 rewriter.replaceOpWithNewOp<BroadcastOp>(
790 op, args: TypeRange{op.getType()}, args&: newShapeOperands, args&: op.getProperties(),
791 args: op->getDiscardableAttrDictionary().getValue());
792 return success();
793 }
794};
795
796template <typename OpTy>
797struct CanonicalizeCastExtentTensorOperandsPattern
798 : public OpRewritePattern<OpTy> {
799 using OpRewritePattern<OpTy>::OpRewritePattern;
800
801 LogicalResult matchAndRewrite(OpTy op,
802 PatternRewriter &rewriter) const override {
803 // Canonicalize operands.
804 bool anyChange = false;
805 auto canonicalizeOperand = [&](Value operand) -> Value {
806 if (auto castOp = operand.getDefiningOp<tensor::CastOp>()) {
807 // Only eliminate the cast if it holds no shape information.
808 bool isInformationLoosingCast =
809 llvm::cast<RankedTensorType>(Val: castOp.getType()).isDynamicDim(idx: 0);
810 if (isInformationLoosingCast) {
811 anyChange = true;
812 return castOp.getSource();
813 }
814 }
815 return operand;
816 };
817 auto newOperands =
818 llvm::map_to_vector<8>(op.getOperands(), canonicalizeOperand);
819
820 // Rewrite op if any change required.
821 if (!anyChange)
822 return failure();
823 rewriter.replaceOpWithNewOp<OpTy>(
824 op, op->getResultTypes(), newOperands, op.getProperties(),
825 op->getDiscardableAttrDictionary().getValue());
826 return success();
827 }
828};
829
830struct BroadcastConcretizeResultTypePattern
831 : public OpRewritePattern<BroadcastOp> {
832 using OpRewritePattern<BroadcastOp>::OpRewritePattern;
833
834 LogicalResult matchAndRewrite(BroadcastOp op,
835 PatternRewriter &rewriter) const override {
836 // Only concretize dynamic extent tensor result types.
837 auto resultTy = llvm::dyn_cast<RankedTensorType>(Val: op.getType());
838 if (!resultTy || !resultTy.isDynamicDim(idx: 0))
839 return failure();
840
841 // Infer resulting shape rank if possible.
842 int64_t maxRank = 0;
843 for (Value shape : op.getShapes()) {
844 if (auto extentTensorTy =
845 llvm::dyn_cast<RankedTensorType>(Val: shape.getType())) {
846 // Cannot infer resulting shape rank if any operand is dynamically
847 // ranked.
848 if (extentTensorTy.isDynamicDim(idx: 0))
849 return failure();
850 maxRank = std::max(a: maxRank, b: extentTensorTy.getDimSize(idx: 0));
851 }
852 }
853
854 auto newOp = BroadcastOp::create(builder&: rewriter, location: op.getLoc(),
855 result: getExtentTensorType(ctx: getContext(), rank: maxRank),
856 shapes: op.getShapes(), /*error=*/nullptr);
857 rewriter.replaceOpWithNewOp<tensor::CastOp>(op, args: op.getType(), args&: newOp);
858 return success();
859 }
860};
861} // namespace
862
863void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
864 MLIRContext *context) {
865 patterns.add<BroadcastConcretizeResultTypePattern,
866 BroadcastFoldConstantOperandsPattern,
867 BroadcastForwardSingleOperandPattern,
868 CanonicalizeCastExtentTensorOperandsPattern<BroadcastOp>,
869 RemoveDuplicateOperandsPattern<BroadcastOp>,
870 RemoveEmptyShapeOperandsPattern<BroadcastOp>>(arg&: context);
871}
872
873//===----------------------------------------------------------------------===//
874// ConcatOp
875//===----------------------------------------------------------------------===//
876
877OpFoldResult ConcatOp::fold(FoldAdaptor adaptor) {
878 // concat with a poison operand is poison.
879 if (isa_and_present<ub::PoisonAttr>(Val: adaptor.getLhs()) ||
880 isa_and_present<ub::PoisonAttr>(Val: adaptor.getRhs()))
881 return ub::PoisonAttr::get(ctx: getContext());
882 auto lhs = llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getLhs());
883 auto rhs = llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getRhs());
884 if (!lhs || !rhs)
885 return nullptr;
886 auto lhsShape = llvm::to_vector<6>(Range: lhs.getValues<int64_t>());
887 auto rhsShape = llvm::to_vector<6>(Range: rhs.getValues<int64_t>());
888 SmallVector<int64_t, 6> resultShape;
889 resultShape.append(in_start: lhsShape.begin(), in_end: lhsShape.end());
890 resultShape.append(in_start: rhsShape.begin(), in_end: rhsShape.end());
891 Builder builder(getContext());
892 return builder.getIndexTensorAttr(values: resultShape);
893}
894
895//===----------------------------------------------------------------------===//
896// ConstShapeOp
897//===----------------------------------------------------------------------===//
898
899void ConstShapeOp::print(OpAsmPrinter &p) {
900 p << " ";
901 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
902 p << "[";
903 interleaveComma(c: getShape().getValues<int64_t>(), os&: p);
904 p << "] : ";
905 p.printType(type: getType());
906}
907
908ParseResult ConstShapeOp::parse(OpAsmParser &parser, OperationState &result) {
909 if (parser.parseOptionalAttrDict(result&: result.attributes))
910 return failure();
911 // We piggy-back on ArrayAttr parsing, though we don't internally store the
912 // shape as an ArrayAttr.
913 // TODO: Implement custom parser and maybe make syntax a bit more concise.
914 Attribute extentsRaw;
915 NamedAttrList dummy;
916 if (parser.parseAttribute(result&: extentsRaw, attrName: "dummy", attrs&: dummy))
917 return failure();
918 auto extentsArray = llvm::dyn_cast<ArrayAttr>(Val&: extentsRaw);
919 if (!extentsArray)
920 return failure();
921 SmallVector<int64_t, 6> ints;
922 for (Attribute extent : extentsArray) {
923 IntegerAttr attr = llvm::dyn_cast<IntegerAttr>(Val&: extent);
924 if (!attr)
925 return failure();
926 ints.push_back(Elt: attr.getInt());
927 }
928 Builder &builder = parser.getBuilder();
929 result.addAttribute(name: "shape", attr: builder.getIndexTensorAttr(values: ints));
930 Type resultTy;
931 if (parser.parseColonType(result&: resultTy))
932 return failure();
933 result.types.push_back(Elt: resultTy);
934 return success();
935}
936
937OpFoldResult ConstShapeOp::fold(FoldAdaptor) { return getShapeAttr(); }
938
939void ConstShapeOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
940 MLIRContext *context) {
941 patterns.add<TensorCastConstShape>(context);
942}
943
944LogicalResult mlir::shape::ConstShapeOp::inferReturnTypes(
945 MLIRContext *context, std::optional<Location> location,
946 ConstShapeOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
947 Builder b(context);
948 const Properties prop = adaptor.getProperties();
949 inferredReturnTypes.assign(IL: {RankedTensorType::get(
950 shape: {static_cast<int64_t>(prop.shape.size())}, elementType: b.getIndexType())});
951 return success();
952}
953
954bool mlir::shape::ConstShapeOp::isCompatibleReturnTypes(TypeRange l,
955 TypeRange r) {
956 if (l.size() != 1 || r.size() != 1)
957 return false;
958
959 Type lhs = l.front();
960 Type rhs = r.front();
961
962 if (llvm::isa<ShapeType>(Val: lhs) || llvm::isa<ShapeType>(Val: rhs))
963 // Shape type is compatible with all other valid return types.
964 return true;
965 return lhs == rhs;
966}
967
968//===----------------------------------------------------------------------===//
969// CstrBroadcastableOp
970//===----------------------------------------------------------------------===//
971
972void CstrBroadcastableOp::getCanonicalizationPatterns(
973 RewritePatternSet &patterns, MLIRContext *context) {
974 // Canonicalization patterns have overlap with the considerations during
975 // folding in case additional shape information is inferred at some point that
976 // does not result in folding.
977 patterns.add<CanonicalizeCastExtentTensorOperandsPattern<CstrBroadcastableOp>,
978 CstrBroadcastableEqOps,
979 RemoveDuplicateOperandsPattern<CstrBroadcastableOp>,
980 RemoveEmptyShapeOperandsPattern<CstrBroadcastableOp>>(arg&: context);
981}
982
983// Return true if there is exactly one attribute not representing a scalar
984// broadcast.
985static bool hasAtMostSingleNonScalar(ArrayRef<Attribute> attributes) {
986 bool nonScalarSeen = false;
987 for (Attribute a : attributes) {
988 // Treat non-constant and poison operands as potentially non-scalar.
989 auto dense = llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val&: a);
990 if (!dense || dense.getNumElements() != 0) {
991 if (nonScalarSeen)
992 return false;
993 nonScalarSeen = true;
994 }
995 }
996 return true;
997}
998
999OpFoldResult CstrBroadcastableOp::fold(FoldAdaptor adaptor) {
1000 // No broadcasting is needed if all operands but one are scalar.
1001 if (hasAtMostSingleNonScalar(attributes: adaptor.getShapes()))
1002 return BoolAttr::get(context: getContext(), value: true);
1003
1004 if ([&] {
1005 SmallVector<SmallVector<int64_t, 6>, 6> extents;
1006 for (const auto &operand : adaptor.getShapes()) {
1007 auto dense = llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: operand);
1008 if (!dense)
1009 return false;
1010 extents.push_back(Elt: llvm::to_vector<6>(Range: dense.getValues<int64_t>()));
1011 }
1012 return OpTrait::util::staticallyKnownBroadcastable(shapes: extents);
1013 }())
1014 return BoolAttr::get(context: getContext(), value: true);
1015
1016 // Lastly, see if folding can be completed based on what constraints are known
1017 // on the input shapes.
1018 if ([&] {
1019 SmallVector<SmallVector<int64_t, 6>, 6> extents;
1020 for (auto shapeValue : getShapes()) {
1021 extents.emplace_back();
1022 if (failed(Result: getShapeVec(input: shapeValue, shapeValues&: extents.back())))
1023 return false;
1024 }
1025 return OpTrait::util::staticallyKnownBroadcastable(shapes: extents);
1026 }())
1027 return BoolAttr::get(context: getContext(), value: true);
1028
1029 // Because a failing witness result here represents an eventual assertion
1030 // failure, we do not replace it with a constant witness.
1031 return nullptr;
1032}
1033
1034LogicalResult CstrBroadcastableOp::verify() {
1035 // Ensure that CstrBroadcastableOp contains at least two operands
1036 if (getNumOperands() < 2)
1037 return emitOpError(message: "required at least 2 input shapes");
1038 return success();
1039}
1040
1041//===----------------------------------------------------------------------===//
1042// CstrEqOp
1043//===----------------------------------------------------------------------===//
1044
1045void CstrEqOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1046 MLIRContext *context) {
1047 // If inputs are equal, return passing witness
1048 patterns.add<CstrEqEqOps>(arg&: context);
1049}
1050
1051OpFoldResult CstrEqOp::fold(FoldAdaptor adaptor) {
1052 if (llvm::all_of(Range: adaptor.getShapes(), P: [&](Attribute a) {
1053 return a && a == adaptor.getShapes().front();
1054 }))
1055 return BoolAttr::get(context: getContext(), value: true);
1056
1057 // Because a failing witness result here represents an eventual assertion
1058 // failure, we do not try to replace it with a constant witness. Similarly, we
1059 // cannot if there are any non-const inputs.
1060 return nullptr;
1061}
1062
1063//===----------------------------------------------------------------------===//
1064// ConstSizeOp
1065//===----------------------------------------------------------------------===//
1066
1067void ConstSizeOp::build(OpBuilder &builder, OperationState &result,
1068 int64_t value) {
1069 build(odsBuilder&: builder, odsState&: result, value: builder.getIndexAttr(value));
1070}
1071
1072OpFoldResult ConstSizeOp::fold(FoldAdaptor) { return getValueAttr(); }
1073
1074void ConstSizeOp::getAsmResultNames(
1075 llvm::function_ref<void(Value, StringRef)> setNameFn) {
1076 SmallString<4> buffer;
1077 llvm::raw_svector_ostream os(buffer);
1078 os << "c" << getValue();
1079 setNameFn(getResult(), os.str());
1080}
1081
1082//===----------------------------------------------------------------------===//
1083// ConstWitnessOp
1084//===----------------------------------------------------------------------===//
1085
1086OpFoldResult ConstWitnessOp::fold(FoldAdaptor) { return getPassingAttr(); }
1087
1088//===----------------------------------------------------------------------===//
1089// CstrRequireOp
1090//===----------------------------------------------------------------------===//
1091
1092OpFoldResult CstrRequireOp::fold(FoldAdaptor adaptor) {
1093 return adaptor.getPred();
1094}
1095
1096//===----------------------------------------------------------------------===//
1097// DimOp
1098//===----------------------------------------------------------------------===//
1099
1100std::optional<int64_t> DimOp::getConstantIndex() {
1101 if (auto constSizeOp = getIndex().getDefiningOp<ConstSizeOp>())
1102 return constSizeOp.getValue().getLimitedValue();
1103 if (auto constantOp = getIndex().getDefiningOp<arith::ConstantOp>())
1104 return llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt();
1105 return std::nullopt;
1106}
1107
1108OpFoldResult DimOp::fold(FoldAdaptor adaptor) {
1109 Type valType = getValue().getType();
1110 auto valShapedType = llvm::dyn_cast<ShapedType>(Val&: valType);
1111 if (!valShapedType || !valShapedType.hasRank())
1112 return nullptr;
1113 std::optional<int64_t> index = getConstantIndex();
1114 if (!index.has_value())
1115 return nullptr;
1116 if (index.value() < 0 || index.value() >= valShapedType.getRank())
1117 return nullptr;
1118 auto extent = valShapedType.getDimSize(idx: *index);
1119 if (ShapedType::isDynamic(dValue: extent))
1120 return nullptr;
1121 return IntegerAttr::get(type: IndexType::get(context: getContext()), value: extent);
1122}
1123
1124LogicalResult mlir::shape::DimOp::inferReturnTypes(
1125 MLIRContext *context, std::optional<Location> location,
1126 DimOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1127 inferredReturnTypes.assign(IL: {adaptor.getIndex().getType()});
1128 return success();
1129}
1130
1131bool mlir::shape::DimOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1132 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1133}
1134
1135//===----------------------------------------------------------------------===//
1136// DivOp
1137//===----------------------------------------------------------------------===//
1138
1139OpFoldResult DivOp::fold(FoldAdaptor adaptor) {
1140 auto lhs = llvm::dyn_cast_if_present<IntegerAttr>(Val: adaptor.getLhs());
1141 if (!lhs)
1142 return nullptr;
1143 auto rhs = llvm::dyn_cast_if_present<IntegerAttr>(Val: adaptor.getRhs());
1144 if (!rhs || rhs.getValue().isZero())
1145 return nullptr;
1146
1147 // Division in APInt does not follow floor(lhs, rhs) when the result is
1148 // negative. Rather, APInt rounds toward zero.
1149 APInt quotient, remainder;
1150 APInt::sdivrem(LHS: lhs.getValue(), RHS: rhs.getValue(), Quotient&: quotient, Remainder&: remainder);
1151 if (quotient.isNegative() && !remainder.isZero()) {
1152 quotient -= 1;
1153 }
1154
1155 Type indexTy = IndexType::get(context: getContext());
1156 return IntegerAttr::get(type: indexTy, value: quotient);
1157}
1158
1159LogicalResult mlir::shape::DivOp::inferReturnTypes(
1160 MLIRContext *context, std::optional<Location> location,
1161 DivOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1162 if (llvm::isa<SizeType>(Val: adaptor.getLhs().getType()) ||
1163 llvm::isa<SizeType>(Val: adaptor.getRhs().getType()))
1164 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1165 else
1166 inferredReturnTypes.assign(IL: {IndexType::get(context)});
1167 return success();
1168}
1169
1170bool mlir::shape::DivOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1171 // SizeType is compatible with IndexType.
1172 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1173}
1174
1175LogicalResult DivOp::verify() { return verifySizeOrIndexOp(op: *this); }
1176
1177//===----------------------------------------------------------------------===//
1178// ShapeEqOp
1179//===----------------------------------------------------------------------===//
1180
1181OpFoldResult ShapeEqOp::fold(FoldAdaptor adaptor) {
1182 bool allSame = true;
1183 if (!adaptor.getShapes().empty() && !adaptor.getShapes().front())
1184 return {};
1185 for (Attribute operand : adaptor.getShapes().drop_front()) {
1186 if (!operand)
1187 return {};
1188 allSame = allSame && operand == adaptor.getShapes().front();
1189 }
1190 return BoolAttr::get(context: getContext(), value: allSame);
1191}
1192
1193//===----------------------------------------------------------------------===//
1194// IndexToSizeOp
1195//===----------------------------------------------------------------------===//
1196
1197OpFoldResult IndexToSizeOp::fold(FoldAdaptor adaptor) {
1198 // Constant values of both types, `shape.size` and `index`, are represented as
1199 // `IntegerAttr`s which makes constant folding simple.
1200 if (Attribute arg = adaptor.getArg())
1201 return arg;
1202 return {};
1203}
1204
1205void IndexToSizeOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1206 MLIRContext *context) {
1207 patterns.add<SizeToIndexToSizeCanonicalization>(context);
1208}
1209
1210//===----------------------------------------------------------------------===//
1211// FromExtentsOp
1212//===----------------------------------------------------------------------===//
1213
1214OpFoldResult FromExtentsOp::fold(FoldAdaptor adaptor) {
1215 SmallVector<int64_t, 6> extents;
1216 for (Attribute attr : adaptor.getExtents()) {
1217 auto intAttr = llvm::dyn_cast_if_present<IntegerAttr>(Val&: attr);
1218 if (!intAttr)
1219 return nullptr;
1220 extents.push_back(Elt: intAttr.getInt());
1221 }
1222 Builder builder(getContext());
1223 return builder.getIndexTensorAttr(values: extents);
1224}
1225
1226//===----------------------------------------------------------------------===//
1227// FunctionLibraryOp
1228//===----------------------------------------------------------------------===//
1229
1230void FunctionLibraryOp::build(OpBuilder &builder, OperationState &result,
1231 StringRef name) {
1232 result.getOrAddProperties<Properties>().sym_name =
1233 builder.getStringAttr(bytes: name);
1234}
1235
1236FuncOp FunctionLibraryOp::getShapeFunction(Operation *op) {
1237 auto attr = llvm::dyn_cast_or_null<FlatSymbolRefAttr>(
1238 Val: getMapping().get(name: op->getName().getIdentifier()));
1239 if (!attr)
1240 return nullptr;
1241 return lookupSymbol<FuncOp>(symbol: attr);
1242}
1243
1244ParseResult FunctionLibraryOp::parse(OpAsmParser &parser,
1245 OperationState &result) {
1246 // Parse the op name.
1247 StringAttr nameAttr;
1248 if (parser.parseSymbolName(result&: nameAttr, attrName: getSymNameAttrName(name: result.name),
1249 attrs&: result.attributes))
1250 return failure();
1251
1252 if (parser.parseOptionalAttrDictWithKeyword(result&: result.attributes))
1253 return failure();
1254
1255 auto *bodyRegion = result.addRegion();
1256 if (parser.parseRegion(region&: *bodyRegion))
1257 return failure();
1258
1259 if (parser.parseKeyword(keyword: "mapping"))
1260 return failure();
1261
1262 DictionaryAttr mappingAttr;
1263 if (parser.parseAttribute(result&: mappingAttr,
1264 type: parser.getBuilder().getType<NoneType>(), attrName: "mapping",
1265 attrs&: result.attributes))
1266 return failure();
1267 return success();
1268}
1269
1270void FunctionLibraryOp::print(OpAsmPrinter &p) {
1271 p << ' ';
1272 p.printSymbolName(symbolRef: getName());
1273 p.printOptionalAttrDictWithKeyword(
1274 attrs: (*this)->getDiscardableAttrDictionary().getValue());
1275 p << ' ';
1276 p.printRegion(blocks&: getRegion(), /*printEntryBlockArgs=*/false,
1277 /*printBlockTerminators=*/false);
1278 p << " mapping ";
1279 p.printAttributeWithoutType(attr: getMappingAttr());
1280}
1281
1282//===----------------------------------------------------------------------===//
1283// FuncOp
1284//===----------------------------------------------------------------------===//
1285
1286FuncOp FuncOp::create(Location location, StringRef name, FunctionType type,
1287 ArrayRef<NamedAttribute> attrs) {
1288 OpBuilder builder(location->getContext());
1289 OperationState state(location, getOperationName());
1290 FuncOp::build(odsBuilder&: builder, odsState&: state, name, type, attrs);
1291 return cast<FuncOp>(Val: Operation::create(state));
1292}
1293FuncOp FuncOp::create(Location location, StringRef name, FunctionType type,
1294 Operation::dialect_attr_range attrs) {
1295 SmallVector<NamedAttribute, 8> attrRef(attrs);
1296 return create(location, name, type, attrs: llvm::ArrayRef(attrRef));
1297}
1298FuncOp FuncOp::create(Location location, StringRef name, FunctionType type,
1299 ArrayRef<NamedAttribute> attrs,
1300 ArrayRef<DictionaryAttr> argAttrs) {
1301 FuncOp func = create(location, name, type, attrs);
1302 func.setAllArgAttrs(argAttrs);
1303 return func;
1304}
1305
1306void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
1307 FunctionType type, ArrayRef<NamedAttribute> attrs,
1308 ArrayRef<DictionaryAttr> argAttrs) {
1309 state.addAttribute(name: FuncOp::getSymNameAttrName(name: state.name),
1310 attr: builder.getStringAttr(bytes: name));
1311 state.addAttribute(name: FuncOp::getFunctionTypeAttrName(name: state.name),
1312 attr: TypeAttr::get(type));
1313 state.attributes.append(inStart: attrs.begin(), inEnd: attrs.end());
1314 state.addRegion();
1315
1316 if (argAttrs.empty())
1317 return;
1318 assert(type.getNumInputs() == argAttrs.size());
1319 call_interface_impl::addArgAndResultAttrs(
1320 builder, result&: state, argAttrs, /*resultAttrs=*/{},
1321 argAttrsName: getArgAttrsAttrName(name: state.name), resAttrsName: getResAttrsAttrName(name: state.name));
1322}
1323
1324ParseResult FuncOp::parse(OpAsmParser &parser, OperationState &result) {
1325 auto buildFuncType =
1326 [](Builder &builder, ArrayRef<Type> argTypes, ArrayRef<Type> results,
1327 function_interface_impl::VariadicFlag,
1328 std::string &) { return builder.getFunctionType(inputs: argTypes, results); };
1329
1330 return function_interface_impl::parseFunctionOp(
1331 parser, result, /*allowVariadic=*/false,
1332 typeAttrName: getFunctionTypeAttrName(name: result.name), funcTypeBuilder: buildFuncType,
1333 argAttrsName: getArgAttrsAttrName(name: result.name), resAttrsName: getResAttrsAttrName(name: result.name));
1334}
1335
1336void FuncOp::print(OpAsmPrinter &p) {
1337 function_interface_impl::printFunctionOp(
1338 p, op: *this, /*isVariadic=*/false, typeAttrName: getFunctionTypeAttrName(),
1339 argAttrsName: getArgAttrsAttrName(), resAttrsName: getResAttrsAttrName());
1340}
1341
1342//===----------------------------------------------------------------------===//
1343// GetExtentOp
1344//===----------------------------------------------------------------------===//
1345
1346std::optional<int64_t> GetExtentOp::getConstantDim() {
1347 if (auto constSizeOp = getDim().getDefiningOp<ConstSizeOp>())
1348 return constSizeOp.getValue().getLimitedValue();
1349 if (auto constantOp = getDim().getDefiningOp<arith::ConstantOp>())
1350 return llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt();
1351 return std::nullopt;
1352}
1353
1354OpFoldResult GetExtentOp::fold(FoldAdaptor adaptor) {
1355 auto elements =
1356 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getShape());
1357 if (!elements)
1358 return nullptr;
1359 std::optional<int64_t> dim = getConstantDim();
1360 if (!dim.has_value())
1361 return nullptr;
1362 if (dim.value() >= elements.getNumElements())
1363 return nullptr;
1364 if (dim.value() < 0)
1365 return nullptr;
1366 return elements.getValues<Attribute>()[(uint64_t)dim.value()];
1367}
1368
1369void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
1370 int64_t dim) {
1371 auto loc = result.location;
1372 auto dimAttr = builder.getIndexAttr(value: dim);
1373 if (llvm::isa<ShapeType>(Val: shape.getType())) {
1374 Value dim = ConstSizeOp::create(builder, location: loc, value: dimAttr);
1375 build(odsBuilder&: builder, odsState&: result, extent: builder.getType<SizeType>(), shape, dim);
1376 } else {
1377 Value dim = arith::ConstantOp::create(builder, location: loc, result: builder.getIndexType(),
1378 value: dimAttr);
1379 build(odsBuilder&: builder, odsState&: result, extent: builder.getIndexType(), shape, dim);
1380 }
1381}
1382
1383LogicalResult mlir::shape::GetExtentOp::inferReturnTypes(
1384 MLIRContext *context, std::optional<Location> location,
1385 GetExtentOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1386 inferredReturnTypes.assign(IL: {IndexType::get(context)});
1387 return success();
1388}
1389
1390bool mlir::shape::GetExtentOp::isCompatibleReturnTypes(TypeRange l,
1391 TypeRange r) {
1392 // SizeType is compatible with IndexType.
1393 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1394}
1395
1396LogicalResult GetExtentOp::verify() { return verifySizeOrIndexOp(op: *this); }
1397
1398//===----------------------------------------------------------------------===//
1399// IsBroadcastableOp
1400//===----------------------------------------------------------------------===//
1401
1402void IsBroadcastableOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1403 MLIRContext *context) {
1404 patterns.add<RemoveDuplicateOperandsPattern<IsBroadcastableOp>>(arg&: context);
1405}
1406
1407OpFoldResult IsBroadcastableOp::fold(FoldAdaptor adaptor) {
1408 // Can always broadcast fewer than two shapes.
1409 if (adaptor.getShapes().size() < 2) {
1410 return BoolAttr::get(context: getContext(), value: true);
1411 }
1412
1413 return nullptr;
1414}
1415
1416//===----------------------------------------------------------------------===//
1417// MeetOp
1418//===----------------------------------------------------------------------===//
1419
1420LogicalResult mlir::shape::MeetOp::inferReturnTypes(
1421 MLIRContext *context, std::optional<Location> location,
1422 MeetOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1423 if (adaptor.getOperands().empty())
1424 return failure();
1425
1426 auto isShapeType = [](Type arg) {
1427 if (llvm::isa<ShapeType>(Val: arg))
1428 return true;
1429 return isExtentTensorType(type: arg);
1430 };
1431
1432 ValueRange::type_range types = adaptor.getOperands().getTypes();
1433 Type acc = types.front();
1434 for (auto t : drop_begin(RangeOrContainer&: types)) {
1435 Type l = acc, r = t;
1436 if (!llvm::isa<ShapeType, SizeType>(Val: l))
1437 std::swap(a&: l, b&: r);
1438
1439 // Handle sizes, propagate error type if present.
1440 if (llvm::isa<SizeType>(Val: l)) {
1441 if (llvm::isa<SizeType, IndexType>(Val: r))
1442 acc = l;
1443 else
1444 return emitOptionalError(loc: location, args: "requires all sizes or shapes");
1445 } else if (llvm::isa<IndexType>(Val: l)) {
1446 if (llvm::isa<IndexType>(Val: r))
1447 acc = r;
1448 else
1449 return emitOptionalError(loc: location, args: "requires all sizes or shapes");
1450 } else if (llvm::isa<ShapeType>(Val: l)) {
1451 // Handle shapes, propagate error type if present.
1452 if (isShapeType(r))
1453 acc = l;
1454 else
1455 return emitOptionalError(loc: location, args: "requires all sizes or shapes");
1456 } else if (isExtentTensorType(type: l)) {
1457 auto rank1 = llvm::cast<RankedTensorType>(Val&: l).getShape()[0];
1458 auto rank2 = llvm::cast<RankedTensorType>(Val&: r).getShape()[0];
1459 if (ShapedType::isDynamic(dValue: rank1))
1460 acc = l;
1461 else if (ShapedType::isDynamic(dValue: rank2))
1462 acc = r;
1463 else if (rank1 != rank2)
1464 return emitOptionalError(loc: location, args: "unequal shape cardinality");
1465 else
1466 acc = l;
1467 }
1468 }
1469 inferredReturnTypes.assign(IL: {acc});
1470 return success();
1471}
1472
1473bool mlir::shape::MeetOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1474 if (l.size() != 1 || r.size() != 1)
1475 return false;
1476 if (l == r)
1477 return true;
1478
1479 Type lhs = l.front();
1480 Type rhs = r.front();
1481
1482 if (!llvm::isa<ShapeType, SizeType>(Val: lhs))
1483 std::swap(a&: lhs, b&: rhs);
1484
1485 if (llvm::isa<SizeType>(Val: lhs))
1486 return llvm::isa<SizeType, IndexType>(Val: rhs);
1487 if (llvm::isa<ShapeType>(Val: lhs))
1488 return llvm::isa<ShapeType, TensorType>(Val: rhs);
1489
1490 if (succeeded(Result: verifyCompatibleShapes(types: {lhs, rhs})))
1491 return true;
1492 return false;
1493}
1494
1495//===----------------------------------------------------------------------===//
1496// RankOp
1497//===----------------------------------------------------------------------===//
1498
1499OpFoldResult shape::RankOp::fold(FoldAdaptor adaptor) {
1500 auto shape =
1501 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getShape());
1502 if (!shape)
1503 return {};
1504 int64_t rank = shape.getNumElements();
1505 Builder builder(getContext());
1506 return builder.getIndexAttr(value: rank);
1507}
1508
1509/// Evaluate the `rank` operation for shapes of ranked tensors at compile time.
1510/// Constant folding fails in cases where only the rank is constant, not the
1511/// shape itself.
1512/// This canonicalization matches `shape.rank(shape.shape_of(%ranked_tensor))`.
1513///
1514/// Example:
1515///
1516/// %shape = shape.shape_of %ranked_tensor : tensor<1x2x?xf32>
1517/// %rank = shape.rank %shape
1518///
1519/// becomes
1520///
1521/// %rank = shape.const_size 3
1522
1523namespace {
1524struct RankShapeOfCanonicalizationPattern
1525 : public OpRewritePattern<shape::RankOp> {
1526 using OpRewritePattern<shape::RankOp>::OpRewritePattern;
1527
1528 LogicalResult matchAndRewrite(shape::RankOp op,
1529 PatternRewriter &rewriter) const override {
1530 auto shapeOfOp = op.getShape().getDefiningOp<ShapeOfOp>();
1531 if (!shapeOfOp)
1532 return failure();
1533 auto rankedTensorType =
1534 llvm::dyn_cast<RankedTensorType>(Val: shapeOfOp.getArg().getType());
1535 if (!rankedTensorType)
1536 return failure();
1537 int64_t rank = rankedTensorType.getRank();
1538 if (llvm::isa<IndexType>(Val: op.getType())) {
1539 rewriter.replaceOpWithNewOp<arith::ConstantIndexOp>(op: op.getOperation(),
1540 args&: rank);
1541 } else if (llvm::isa<shape::SizeType>(Val: op.getType())) {
1542 rewriter.replaceOpWithNewOp<shape::ConstSizeOp>(op: op.getOperation(), args&: rank);
1543 } else {
1544 return failure();
1545 }
1546 return success();
1547 }
1548};
1549} // namespace
1550
1551void shape::RankOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1552 MLIRContext *context) {
1553 patterns.add<RankShapeOfCanonicalizationPattern>(arg&: context);
1554}
1555
1556LogicalResult mlir::shape::RankOp::inferReturnTypes(
1557 MLIRContext *context, std::optional<Location> location,
1558 RankOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1559 if (llvm::isa<ShapeType>(Val: adaptor.getShape().getType()))
1560 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1561 else
1562 inferredReturnTypes.assign(IL: {IndexType::get(context)});
1563 return success();
1564}
1565
1566bool mlir::shape::RankOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1567 // SizeType is compatible with IndexType.
1568 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1569}
1570
1571LogicalResult shape::RankOp::verify() { return verifySizeOrIndexOp(op: *this); }
1572
1573//===----------------------------------------------------------------------===//
1574// NumElementsOp
1575//===----------------------------------------------------------------------===//
1576
1577OpFoldResult NumElementsOp::fold(FoldAdaptor adaptor) {
1578 // num_elements of poison shape is poison.
1579 if (isa_and_present<ub::PoisonAttr>(Val: adaptor.getShape()))
1580 return ub::PoisonAttr::get(ctx: getContext());
1581
1582 // Fold only when argument constant.
1583 auto shape =
1584 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getShape());
1585 if (!shape)
1586 return {};
1587
1588 APInt product(64, 1);
1589 for (auto value : shape)
1590 product *= value;
1591 Builder builder(getContext());
1592 return builder.getIndexAttr(value: product.getLimitedValue());
1593}
1594
1595LogicalResult mlir::shape::NumElementsOp::inferReturnTypes(
1596 MLIRContext *context, std::optional<Location> location,
1597 NumElementsOp::Adaptor adaptor,
1598 SmallVectorImpl<Type> &inferredReturnTypes) {
1599 if (llvm::isa<ShapeType>(Val: adaptor.getShape().getType()))
1600 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1601 else
1602 inferredReturnTypes.assign(IL: {IndexType::get(context)});
1603 return success();
1604}
1605
1606bool mlir::shape::NumElementsOp::isCompatibleReturnTypes(TypeRange l,
1607 TypeRange r) {
1608 // SizeType is compatible with IndexType.
1609 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1610}
1611
1612LogicalResult shape::NumElementsOp::verify() {
1613 return verifySizeOrIndexOp(op: *this);
1614}
1615
1616//===----------------------------------------------------------------------===//
1617// MaxOp
1618//===----------------------------------------------------------------------===//
1619
1620OpFoldResult MaxOp::fold(FoldAdaptor adaptor) {
1621 // If operands are equal, just propagate one.
1622 if (getLhs() == getRhs())
1623 return getLhs();
1624 return nullptr;
1625}
1626
1627LogicalResult mlir::shape::MaxOp::inferReturnTypes(
1628 MLIRContext *context, std::optional<Location> location,
1629 MaxOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1630 if (adaptor.getLhs().getType() == adaptor.getRhs().getType())
1631 inferredReturnTypes.assign(IL: {adaptor.getLhs().getType()});
1632 else
1633 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1634 return success();
1635}
1636
1637bool mlir::shape::MaxOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1638 if (l.size() != 1 || r.size() != 1)
1639 return false;
1640 if (llvm::isa<ShapeType>(Val: l.front()) && llvm::isa<ShapeType>(Val: r.front()))
1641 return true;
1642 if (llvm::isa<SizeType>(Val: l.front()) && llvm::isa<SizeType>(Val: r.front()))
1643 return true;
1644 return false;
1645}
1646
1647//===----------------------------------------------------------------------===//
1648// MinOp
1649//===----------------------------------------------------------------------===//
1650
1651OpFoldResult MinOp::fold(FoldAdaptor adaptor) {
1652 // If operands are equal, just propagate one.
1653 if (getLhs() == getRhs())
1654 return getLhs();
1655 return nullptr;
1656}
1657
1658LogicalResult mlir::shape::MinOp::inferReturnTypes(
1659 MLIRContext *context, std::optional<Location> location,
1660 MinOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1661 if (adaptor.getLhs().getType() == adaptor.getRhs().getType())
1662 inferredReturnTypes.assign(IL: {adaptor.getLhs().getType()});
1663 else
1664 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1665 return success();
1666}
1667
1668bool mlir::shape::MinOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1669 if (l.size() != 1 || r.size() != 1)
1670 return false;
1671 if (llvm::isa<ShapeType>(Val: l.front()) && llvm::isa<ShapeType>(Val: r.front()))
1672 return true;
1673 if (llvm::isa<SizeType>(Val: l.front()) && llvm::isa<SizeType>(Val: r.front()))
1674 return true;
1675 return false;
1676}
1677
1678//===----------------------------------------------------------------------===//
1679// MulOp
1680//===----------------------------------------------------------------------===//
1681
1682OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
1683 auto lhs = llvm::dyn_cast_if_present<IntegerAttr>(Val: adaptor.getLhs());
1684 if (!lhs)
1685 return nullptr;
1686 auto rhs = llvm::dyn_cast_if_present<IntegerAttr>(Val: adaptor.getRhs());
1687 if (!rhs)
1688 return nullptr;
1689 APInt folded = lhs.getValue() * rhs.getValue();
1690 Type indexTy = IndexType::get(context: getContext());
1691 return IntegerAttr::get(type: indexTy, value: folded);
1692}
1693
1694LogicalResult mlir::shape::MulOp::inferReturnTypes(
1695 MLIRContext *context, std::optional<Location> location,
1696 MulOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1697 if (llvm::isa<SizeType>(Val: adaptor.getLhs().getType()) ||
1698 llvm::isa<SizeType>(Val: adaptor.getRhs().getType()))
1699 inferredReturnTypes.assign(IL: {SizeType::get(ctx: context)});
1700 else
1701 inferredReturnTypes.assign(IL: {IndexType::get(context)});
1702 return success();
1703}
1704
1705bool mlir::shape::MulOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1706 // SizeType is compatible with IndexType.
1707 return eachHasOnlyOneOfTypes<SizeType, IndexType>(l, rs: r);
1708}
1709
1710LogicalResult shape::MulOp::verify() { return verifySizeOrIndexOp(op: *this); }
1711
1712//===----------------------------------------------------------------------===//
1713// ShapeOfOp
1714//===----------------------------------------------------------------------===//
1715
1716namespace {
1717/// Replace shape_of(x) where x has a constant shape with a const_shape op.
1718struct ShapeOfOpToConstShapeOp : public OpRewritePattern<shape::ShapeOfOp> {
1719 using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
1720
1721 LogicalResult matchAndRewrite(shape::ShapeOfOp op,
1722 PatternRewriter &rewriter) const override {
1723 auto type = llvm::dyn_cast<ShapedType>(Val: op.getArg().getType());
1724 if (!type || !type.hasStaticShape())
1725 return failure();
1726
1727 Type resultType = op.getResult().getType();
1728 Location loc = op.getLoc();
1729 Type constResType =
1730 isa<ShapeType>(Val: resultType)
1731 ? resultType
1732 : RankedTensorType::get(shape: {type.getRank()}, elementType: rewriter.getIndexType());
1733 Value constShape =
1734 ConstShapeOp::create(builder&: rewriter, location: loc, result: constResType,
1735 shape: rewriter.getIndexTensorAttr(values: type.getShape()))
1736 .getResult();
1737 if (constShape.getType() != resultType)
1738 constShape =
1739 tensor::CastOp::create(builder&: rewriter, location: loc, dest: resultType, source: constShape);
1740 rewriter.replaceOp(op, newValues: constShape);
1741 return success();
1742 }
1743};
1744
1745// Canonicalize
1746//
1747// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
1748// tensor<*xf32>
1749// %1 = shape.shape_of %0 : tensor<*xf32> -> tensor<?xindex>
1750//
1751// to
1752//
1753// %0 = tensor.reshape %input(%shape) : (tensor<*xf32>, tensor<?xindex>) ->
1754// tensor<*xf32>
1755// %1 = %shape
1756//
1757struct ShapeOfFromReshape : public OpRewritePattern<shape::ShapeOfOp> {
1758 using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
1759
1760 LogicalResult matchAndRewrite(shape::ShapeOfOp op,
1761 PatternRewriter &rewriter) const override {
1762 auto tensorReshapeOp = op.getArg().getDefiningOp<tensor::ReshapeOp>();
1763 if (!tensorReshapeOp)
1764 return rewriter.notifyMatchFailure(arg&: op, msg: "producer is not tensor.reshape");
1765 if (!isa<TensorType>(Val: op.getType()))
1766 return rewriter.notifyMatchFailure(arg&: op, msg: "result is not a tensor");
1767
1768 // Operand 'shape' of 'tensor.reshape' may now be used as the result of
1769 // 'shape.shape_of'. While its type is guaranteed to be compatible in well-
1770 // formed IR, it may not be identical (dynamically vs statically shaped),
1771 // in which case it needs to be cast first using 'tensor.cast'.
1772 // Additionally, it may not have identical element type (i32 vs index)
1773 // while it has identical shaped type (dynamic vs static), in which case it
1774 // needs to be cast first using 'arith.index_cast'. Note: 'shape.shape_of'
1775 // op result must be shape or extent tensor.
1776 Value shape = tensorReshapeOp.getShape();
1777
1778 auto opTensorTy = cast<RankedTensorType>(Val: op.getType());
1779 auto shapeTensorTy = cast<RankedTensorType>(Val: shape.getType());
1780
1781 if (opTensorTy != shapeTensorTy) {
1782 if (opTensorTy.getElementType() == shapeTensorTy.getElementType())
1783 shape =
1784 tensor::CastOp::create(builder&: rewriter, location: op.getLoc(), dest: opTensorTy, source: shape);
1785 else if (!isExtentTensorType(type: shapeTensorTy))
1786 shape = arith::IndexCastOp::create(builder&: rewriter, location: op.getLoc(), out: opTensorTy,
1787 in: shape);
1788 }
1789
1790 rewriter.replaceOp(op, newValues: shape);
1791 return success();
1792 }
1793};
1794
1795// Canonicalize
1796// ```
1797// %0 = shape.shape_of %arg : tensor<?x?x?xf32> -> tensor<3xindex>
1798// %1 = tensor.cast %0 : tensor<3xindex> to tensor<?xindex>
1799// ```
1800// to
1801// ```
1802// %1 = shape.shape_of %arg : tensor<?x?x?xf32> -> tensor<?xindex>
1803// ```
1804struct ShapeOfCastExtentTensor : public OpRewritePattern<tensor::CastOp> {
1805 using OpRewritePattern<tensor::CastOp>::OpRewritePattern;
1806
1807 LogicalResult matchAndRewrite(tensor::CastOp op,
1808 PatternRewriter &rewriter) const override {
1809 auto ty = llvm::dyn_cast<RankedTensorType>(Val: op.getType());
1810 if (!ty || ty.getRank() != 1)
1811 return failure();
1812
1813 auto shapeOfOp = op.getSource().getDefiningOp<ShapeOfOp>();
1814 if (!shapeOfOp)
1815 return failure();
1816
1817 // Argument type must be ranked and must not conflict.
1818 auto argTy = llvm::dyn_cast<RankedTensorType>(Val: shapeOfOp.getArg().getType());
1819 if (!argTy || (!ty.isDynamicDim(idx: 0) && ty.getDimSize(idx: 0) != argTy.getRank()))
1820 return failure();
1821
1822 rewriter.replaceOpWithNewOp<ShapeOfOp>(op, args&: ty, args: shapeOfOp.getArg());
1823 return success();
1824 }
1825};
1826} // namespace
1827
1828void ShapeOfOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1829 MLIRContext *context) {
1830 patterns.add<ShapeOfCastExtentTensor, ShapeOfFromReshape,
1831 ExtractFromShapeOfExtentTensor, ShapeOfOpToConstShapeOp>(
1832 context);
1833}
1834
1835LogicalResult mlir::shape::ShapeOfOp::inferReturnTypes(
1836 MLIRContext *context, std::optional<Location> location,
1837 ShapeOfOp::Adaptor adaptor, SmallVectorImpl<Type> &inferredReturnTypes) {
1838 if (llvm::isa<ValueShapeType>(Val: adaptor.getArg().getType()))
1839 inferredReturnTypes.assign(IL: {ShapeType::get(ctx: context)});
1840 else {
1841 auto shapedTy = llvm::cast<ShapedType>(Val: adaptor.getArg().getType());
1842 int64_t rank =
1843 shapedTy.hasRank() ? shapedTy.getRank() : ShapedType::kDynamic;
1844 Type indexTy = IndexType::get(context);
1845 Type extentTensorTy = RankedTensorType::get(shape: {rank}, elementType: indexTy);
1846 inferredReturnTypes.assign(IL: {extentTensorTy});
1847 }
1848 return success();
1849}
1850
1851bool mlir::shape::ShapeOfOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1852 if (l.size() != 1 || r.size() != 1)
1853 return false;
1854 if (l == r)
1855 return true;
1856
1857 Type lhs = l.front();
1858 Type rhs = r.front();
1859
1860 if (!llvm::isa<ShapeType, ShapedType>(Val: lhs) ||
1861 !llvm::isa<ShapeType, ShapedType>(Val: rhs))
1862 return false;
1863
1864 if (llvm::isa<ShapeType>(Val: lhs) || llvm::isa<ShapeType>(Val: rhs))
1865 // Shape type is compatible with all other valid return types.
1866 return true;
1867
1868 if (succeeded(Result: verifyCompatibleShapes(types: {lhs, rhs})))
1869 return true;
1870 return false;
1871}
1872
1873LogicalResult shape::ShapeOfOp::verify() {
1874 return verifyShapeOrExtentTensorOp(op: *this);
1875}
1876
1877//===----------------------------------------------------------------------===//
1878// SizeToIndexOp
1879//===----------------------------------------------------------------------===//
1880
1881OpFoldResult SizeToIndexOp::fold(FoldAdaptor adaptor) {
1882 // Constant values of both types, `shape.size` and `index`, are represented as
1883 // `IntegerAttr`s which makes constant folding simple.
1884 if (Attribute arg = adaptor.getArg())
1885 return arg;
1886 return OpFoldResult();
1887}
1888
1889void SizeToIndexOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1890 MLIRContext *context) {
1891 patterns.add<IndexToSizeToIndexCanonicalization>(context);
1892}
1893
1894bool SizeToIndexOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1895 if (inputs.size() != 1 || outputs.size() != 1)
1896 return false;
1897 return llvm::isa<IndexType, SizeType>(Val: inputs[0]) &&
1898 llvm::isa<IndexType>(Val: outputs[0]);
1899}
1900
1901//===----------------------------------------------------------------------===//
1902// YieldOp
1903//===----------------------------------------------------------------------===//
1904
1905LogicalResult shape::YieldOp::verify() {
1906 auto *parentOp = (*this)->getParentOp();
1907 auto results = parentOp->getResults();
1908 auto operands = getOperands();
1909
1910 if (parentOp->getNumResults() != getNumOperands())
1911 return emitOpError() << "number of operands does not match number of "
1912 "results of its parent";
1913 for (auto e : llvm::zip(t&: results, u&: operands))
1914 if (std::get<0>(t&: e).getType() != std::get<1>(t&: e).getType())
1915 return emitOpError() << "types mismatch between yield op and its parent";
1916
1917 return success();
1918}
1919
1920//===----------------------------------------------------------------------===//
1921// SplitAtOp
1922//===----------------------------------------------------------------------===//
1923
1924LogicalResult SplitAtOp::fold(FoldAdaptor adaptor,
1925 SmallVectorImpl<OpFoldResult> &results) {
1926 // Splitting a poison shape, or at a poison index, gives two poison halves.
1927 if (isa_and_present<ub::PoisonAttr>(Val: adaptor.getOperand()) ||
1928 isa_and_present<ub::PoisonAttr>(Val: adaptor.getIndex())) {
1929 auto poison = ub::PoisonAttr::get(ctx: getContext());
1930 results.push_back(Elt: poison);
1931 results.push_back(Elt: poison);
1932 return success();
1933 }
1934
1935 auto shapeAttr =
1936 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getOperand());
1937 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(Val: adaptor.getIndex());
1938 if (!shapeAttr || !indexAttr)
1939 return failure();
1940 auto shapeVec = llvm::to_vector<6>(Range: shapeAttr.getValues<int64_t>());
1941 auto shape = llvm::ArrayRef(shapeVec);
1942 auto splitPoint = indexAttr.getInt();
1943 // Verify that the split point is in the correct range.
1944 // TODO: Constant fold to an "error".
1945 int64_t rank = shape.size();
1946 if (-rank > splitPoint || splitPoint > rank)
1947 return failure();
1948 if (splitPoint < 0)
1949 splitPoint += shape.size();
1950 Builder builder(adaptor.getOperand().getContext());
1951 results.push_back(Elt: builder.getIndexTensorAttr(values: shape.take_front(N: splitPoint)));
1952 results.push_back(Elt: builder.getIndexTensorAttr(values: shape.drop_front(N: splitPoint)));
1953 return success();
1954}
1955
1956//===----------------------------------------------------------------------===//
1957// ToExtentTensorOp
1958//===----------------------------------------------------------------------===//
1959
1960OpFoldResult ToExtentTensorOp::fold(FoldAdaptor adaptor) {
1961 // to_extent_tensor of a poison shape is poison.
1962 if (isa_and_present<ub::PoisonAttr>(Val: adaptor.getInput()))
1963 return ub::PoisonAttr::get(ctx: getContext());
1964
1965 auto inputAttr =
1966 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getInput());
1967 if (!inputAttr)
1968 return OpFoldResult();
1969 Builder builder(getContext());
1970 auto shape = llvm::to_vector<6>(Range: inputAttr.getValues<int64_t>());
1971 auto type = RankedTensorType::get(shape: {static_cast<int64_t>(shape.size())},
1972 elementType: builder.getIndexType());
1973 return DenseIntElementsAttr::get(type, arg&: shape);
1974}
1975
1976bool ToExtentTensorOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1977 if (inputs.size() != 1 || outputs.size() != 1)
1978 return false;
1979 if (auto inputTensor = llvm::dyn_cast<RankedTensorType>(Val: inputs[0])) {
1980 if (!llvm::isa<IndexType>(Val: inputTensor.getElementType()) ||
1981 inputTensor.getRank() != 1)
1982 return false;
1983 } else if (!llvm::isa<ShapeType>(Val: inputs[0])) {
1984 return false;
1985 }
1986
1987 TensorType outputTensor = llvm::dyn_cast<TensorType>(Val: outputs[0]);
1988 return outputTensor && llvm::isa<IndexType>(Val: outputTensor.getElementType());
1989}
1990
1991//===----------------------------------------------------------------------===//
1992// ReduceOp
1993//===----------------------------------------------------------------------===//
1994
1995void ReduceOp::build(OpBuilder &builder, OperationState &result, Value shape,
1996 ValueRange initVals) {
1997 OpBuilder::InsertionGuard g(builder);
1998 result.addOperands(newOperands: shape);
1999 result.addOperands(newOperands: initVals);
2000
2001 Region *bodyRegion = result.addRegion();
2002 Block *bodyBlock = builder.createBlock(
2003 parent: bodyRegion, /*insertPt=*/{}, argTypes: builder.getIndexType(), locs: result.location);
2004
2005 Type elementType;
2006 if (auto tensorType = llvm::dyn_cast<TensorType>(Val: shape.getType()))
2007 elementType = tensorType.getElementType();
2008 else
2009 elementType = SizeType::get(ctx: builder.getContext());
2010 bodyBlock->addArgument(type: elementType, loc: shape.getLoc());
2011
2012 for (Value initVal : initVals) {
2013 bodyBlock->addArgument(type: initVal.getType(), loc: initVal.getLoc());
2014 result.addTypes(newTypes: initVal.getType());
2015 }
2016}
2017
2018LogicalResult ReduceOp::verify() {
2019 // Verify block arg types.
2020 Block &block = getRegion().front();
2021
2022 // The block takes index, extent, and aggregated values as arguments.
2023 auto blockArgsCount = getInitVals().size() + 2;
2024 if (block.getNumArguments() != blockArgsCount)
2025 return emitOpError() << "ReduceOp body is expected to have "
2026 << blockArgsCount << " arguments";
2027
2028 // The first block argument is the index and must always be of type `index`.
2029 if (!llvm::isa<IndexType>(Val: block.getArgument(i: 0).getType()))
2030 return emitOpError(
2031 message: "argument 0 of ReduceOp body is expected to be of IndexType");
2032
2033 // The second block argument is the extent and must be of type `size` or
2034 // `index`, depending on whether the reduce operation is applied to a shape or
2035 // to an extent tensor.
2036 Type extentTy = block.getArgument(i: 1).getType();
2037 if (llvm::isa<ShapeType>(Val: getShape().getType())) {
2038 if (!llvm::isa<SizeType>(Val: extentTy))
2039 return emitOpError(message: "argument 1 of ReduceOp body is expected to be of "
2040 "SizeType if the ReduceOp operates on a ShapeType");
2041 } else {
2042 if (!llvm::isa<IndexType>(Val: extentTy))
2043 return emitOpError(
2044 message: "argument 1 of ReduceOp body is expected to be of IndexType if the "
2045 "ReduceOp operates on an extent tensor");
2046 }
2047
2048 for (const auto &type : llvm::enumerate(First: getInitVals()))
2049 if (block.getArgument(i: type.index() + 2).getType() != type.value().getType())
2050 return emitOpError() << "type mismatch between argument "
2051 << type.index() + 2
2052 << " of ReduceOp body and initial value "
2053 << type.index();
2054 return success();
2055}
2056
2057ParseResult ReduceOp::parse(OpAsmParser &parser, OperationState &result) {
2058 // Parse operands.
2059 SmallVector<OpAsmParser::UnresolvedOperand, 3> operands;
2060 Type shapeOrExtentTensorType;
2061 if (parser.parseOperandList(result&: operands, /*requiredOperandCount=*/-1,
2062 delimiter: OpAsmParser::Delimiter::Paren) ||
2063 parser.parseColonType(result&: shapeOrExtentTensorType) ||
2064 parser.parseOptionalArrowTypeList(result&: result.types))
2065 return failure();
2066
2067 // Resolve operands.
2068 auto initVals = llvm::ArrayRef(operands).drop_front();
2069 if (parser.resolveOperand(operand: operands.front(), type: shapeOrExtentTensorType,
2070 result&: result.operands) ||
2071 parser.resolveOperands(operands&: initVals, types&: result.types, loc: parser.getNameLoc(),
2072 result&: result.operands))
2073 return failure();
2074
2075 // Parse the body.
2076 Region *body = result.addRegion();
2077 if (parser.parseRegion(region&: *body, /*args=*/arguments: {}, /*argTypes=*/enableNameShadowing: {}))
2078 return failure();
2079
2080 // Parse attributes.
2081 if (parser.parseOptionalAttrDict(result&: result.attributes))
2082 return failure();
2083
2084 return success();
2085}
2086
2087void ReduceOp::print(OpAsmPrinter &p) {
2088 p << '(' << getShape() << ", " << getInitVals()
2089 << ") : " << getShape().getType();
2090 p.printOptionalArrowTypeList(types: getResultTypes());
2091 p << ' ';
2092 p.printRegion(blocks&: getRegion());
2093 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
2094}
2095
2096#define GET_OP_CLASSES
2097#include "mlir/Dialect/Shape/IR/ShapeOps.cpp.inc"
2098
2099#define GET_TYPEDEF_CLASSES
2100#include "mlir/Dialect/Shape/IR/ShapeOpsTypes.cpp.inc"
2101