1//===- Operation.cpp - Operation support code -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/IR/Operation.h"
10#include "mlir/IR/Attributes.h"
11#include "mlir/IR/BuiltinAttributes.h"
12#include "mlir/IR/BuiltinTypes.h"
13#include "mlir/IR/Dialect.h"
14#include "mlir/IR/IRMapping.h"
15#include "mlir/IR/Matchers.h"
16#include "mlir/IR/OpImplementation.h"
17#include "mlir/IR/OperationSupport.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/IR/TypeUtilities.h"
20#include "mlir/Interfaces/FoldInterfaces.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/Support/ErrorHandling.h"
24#include <optional>
25
26using namespace mlir;
27
28//===----------------------------------------------------------------------===//
29// Operation
30//===----------------------------------------------------------------------===//
31
32/// Create a new Operation from operation state.
33Operation *Operation::create(const OperationState &state) {
34 Operation *op =
35 create(location: state.location, name: state.name, resultTypes: state.types, operands: state.operands,
36 attributes: state.attributes.getDictionary(context: state.getContext()),
37 properties: state.properties, successors: state.successors, regions: state.regions);
38 if (LLVM_UNLIKELY(state.propertiesAttr)) {
39 assert(!state.properties);
40 LogicalResult result =
41 op->setPropertiesFromAttribute(attr: state.propertiesAttr,
42 /*emitError=*/nullptr);
43 assert(result.succeeded() && "invalid properties in op creation");
44 (void)result;
45 }
46 return op;
47}
48
49/// Create a new Operation with the specific fields.
50Operation *Operation::create(Location location, OperationName name,
51 TypeRange resultTypes, ValueRange operands,
52 NamedAttrList &&attributes, PropertyRef properties,
53 BlockRange successors, RegionRange regions) {
54 unsigned numRegions = regions.size();
55 Operation *op =
56 create(location, name, resultTypes, operands, attributes: std::move(attributes),
57 properties, successors, numRegions);
58 for (unsigned i = 0; i < numRegions; ++i)
59 if (regions[i])
60 op->getRegion(index: i).takeBody(other&: *regions[i]);
61 return op;
62}
63
64/// Create a new Operation with the specific fields.
65Operation *Operation::create(Location location, OperationName name,
66 TypeRange resultTypes, ValueRange operands,
67 NamedAttrList &&attributes, PropertyRef properties,
68 BlockRange successors, unsigned numRegions) {
69 // Populate default attributes.
70 name.populateDefaultAttrs(attrs&: attributes);
71
72 return create(location, name, resultTypes, operands,
73 attributes: attributes.getDictionary(context: location.getContext()), properties,
74 successors, numRegions);
75}
76
77/// Overload of create that takes an existing DictionaryAttr to avoid
78/// unnecessarily uniquing a list of attributes.
79Operation *Operation::create(Location location, OperationName name,
80 TypeRange resultTypes, ValueRange operands,
81 DictionaryAttr attributes, PropertyRef properties,
82 BlockRange successors, unsigned numRegions) {
83 assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
84 "unexpected null result type");
85
86 // We only need to allocate additional memory for a subset of results.
87 unsigned numTrailingResults = OpResult::getNumTrailing(numResults: resultTypes.size());
88 unsigned numInlineResults = OpResult::getNumInline(numResults: resultTypes.size());
89 unsigned numSuccessors = successors.size();
90 unsigned numOperands = operands.size();
91 unsigned numResults = resultTypes.size();
92 int opPropertiesAllocSize = llvm::alignTo<8>(Value: name.getOpPropertyByteSize());
93
94 // If the operation is known to have no operands, don't allocate an operand
95 // storage.
96 bool needsOperandStorage =
97 operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true;
98
99 // Compute the byte size for the operation and the operand storage. This takes
100 // into account the size of the operation, its trailing objects, and its
101 // prefixed objects.
102 size_t byteSize =
103 totalSizeToAlloc<detail::OperandStorage, detail::OpProperties,
104 BlockOperand, Region, OpOperand>(
105 Counts: needsOperandStorage ? 1 : 0, Counts: opPropertiesAllocSize, Counts: numSuccessors,
106 Counts: numRegions, Counts: numOperands);
107 size_t prefixByteSize = llvm::alignTo(
108 Value: Operation::prefixAllocSize(numOutOfLineResults: numTrailingResults, numInlineResults),
109 Align: alignof(Operation));
110 char *mallocMem = reinterpret_cast<char *>(malloc(size: byteSize + prefixByteSize));
111 void *rawMem = mallocMem + prefixByteSize;
112
113 // Create the new Operation.
114 Operation *op = ::new (rawMem) Operation(
115 location, name, numResults, numSuccessors, numRegions,
116 opPropertiesAllocSize, attributes, properties, needsOperandStorage);
117
118 assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
119 "unexpected successors in a non-terminator operation");
120
121 // Initialize the results.
122 auto resultTypeIt = resultTypes.begin();
123 for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt)
124 new (op->getInlineOpResult(resultNumber: i)) detail::InlineOpResult(*resultTypeIt, i);
125 for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) {
126 new (op->getOutOfLineOpResult(resultNumber: i))
127 detail::OutOfLineOpResult(*resultTypeIt, i);
128 }
129
130 // Initialize the regions.
131 for (unsigned i = 0; i != numRegions; ++i)
132 new (&op->getRegion(index: i)) Region(op);
133
134 // Initialize the operands.
135 if (needsOperandStorage) {
136 new (&op->getOperandStorage()) detail::OperandStorage(
137 op, op->getTrailingObjects<OpOperand>(), operands);
138 }
139
140 // Initialize the successors.
141 auto blockOperands = op->getBlockOperands();
142 for (unsigned i = 0; i != numSuccessors; ++i)
143 new (&blockOperands[i]) BlockOperand(op, successors[i]);
144
145 // This must be done after properties are initialized.
146 op->setAttrs(attributes);
147
148 return op;
149}
150
151Operation::Operation(Location location, OperationName name, unsigned numResults,
152 unsigned numSuccessors, unsigned numRegions,
153 int fullPropertiesStorageSize, DictionaryAttr attributes,
154 PropertyRef properties, bool hasOperandStorage)
155 : location(location), numResults(numResults), numSuccs(numSuccessors),
156 numRegions(numRegions), hasOperandStorage(hasOperandStorage),
157 propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
158 assert(attributes && "unexpected null attribute dictionary");
159 assert(fullPropertiesStorageSize <= propertiesCapacity &&
160 "Properties size overflow");
161#ifndef NDEBUG
162 if (!getDialect() && !getContext()->allowsUnregisteredDialects())
163 llvm::report_fatal_error(
164 name.getStringRef() +
165 " created with unregistered dialect. If this is intended, please call "
166 "allowUnregisteredDialects() on the MLIRContext, or use "
167 "-allow-unregistered-dialect with the MLIR tool used.");
168#endif
169 if (fullPropertiesStorageSize)
170 name.initOpProperties(storage: getPropertiesStorage(), init: properties);
171}
172
173// Operations are deleted through the destroy() member because they are
174// allocated via malloc.
175Operation::~Operation() {
176 assert(block == nullptr && "operation destroyed but still in a block");
177#ifndef NDEBUG
178 if (!use_empty()) {
179 {
180 InFlightDiagnostic diag =
181 emitOpError("operation destroyed but still has uses");
182 for (Operation *user : getUsers())
183 diag.attachNote(user->getLoc()) << "- use: " << *user << "\n";
184 }
185 llvm::report_fatal_error("operation destroyed but still has uses");
186 }
187#endif
188 // Explicitly run the destructors for the operands.
189 if (hasOperandStorage)
190 getOperandStorage().~OperandStorage();
191
192 // Explicitly run the destructors for the successors.
193 for (auto &successor : getBlockOperands())
194 successor.~BlockOperand();
195
196 // Explicitly destroy the regions.
197 for (auto &region : getRegions())
198 region.~Region();
199 if (propertiesStorageSize)
200 name.destroyOpProperties(properties: getPropertiesStorage());
201}
202
203/// Destroy this operation or one of its subclasses.
204void Operation::destroy() {
205 // Operations may have additional prefixed allocation, which needs to be
206 // accounted for here when computing the address to free.
207 char *rawMem = reinterpret_cast<char *>(this) -
208 llvm::alignTo(Value: prefixAllocSize(), Align: alignof(Operation));
209 this->~Operation();
210 free(ptr: rawMem);
211}
212
213/// Return true if this operation is a proper ancestor of the `other`
214/// operation.
215bool Operation::isProperAncestor(Operation *other) {
216 while ((other = other->getParentOp()))
217 if (this == other)
218 return true;
219 return false;
220}
221
222/// Replace any uses of 'from' with 'to' within this operation.
223void Operation::replaceUsesOfWith(Value from, Value to) {
224 if (from == to)
225 return;
226 for (auto &operand : getOpOperands())
227 if (operand.get() == from)
228 operand.set(to);
229}
230
231/// Replace the current operands of this operation with the ones provided in
232/// 'operands'.
233void Operation::setOperands(ValueRange operands) {
234 if (LLVM_LIKELY(hasOperandStorage))
235 return getOperandStorage().setOperands(owner: this, values: operands);
236 assert(operands.empty() && "setting operands without an operand storage");
237}
238
239/// Replace the operands beginning at 'start' and ending at 'start' + 'length'
240/// with the ones provided in 'operands'. 'operands' may be smaller or larger
241/// than the range pointed to by 'start'+'length'.
242void Operation::setOperands(unsigned start, unsigned length,
243 ValueRange operands) {
244 assert((start + length) <= getNumOperands() &&
245 "invalid operand range specified");
246 if (LLVM_LIKELY(hasOperandStorage))
247 return getOperandStorage().setOperands(owner: this, start, length, operands);
248 assert(operands.empty() && "setting operands without an operand storage");
249}
250
251/// Insert the given operands into the operand list at the given 'index'.
252void Operation::insertOperands(unsigned index, ValueRange operands) {
253 if (LLVM_LIKELY(hasOperandStorage))
254 return setOperands(start: index, /*length=*/0, operands);
255 assert(operands.empty() && "inserting operands without an operand storage");
256}
257
258//===----------------------------------------------------------------------===//
259// Diagnostics
260//===----------------------------------------------------------------------===//
261
262/// Emit an error about fatal conditions with this operation, reporting up to
263/// any diagnostic handlers that may be listening.
264InFlightDiagnostic Operation::emitError(const Twine &message) {
265 InFlightDiagnostic diag = mlir::emitError(loc: getLoc(), message);
266 if (getContext()->shouldPrintOpOnDiagnostic()) {
267 diag.attachNote(noteLoc: getLoc())
268 .append(arg: "see current operation: ")
269 .appendOp(op&: *this, flags: OpPrintingFlags().printGenericOpForm());
270 }
271 return diag;
272}
273
274/// Emit a warning about this operation, reporting up to any diagnostic
275/// handlers that may be listening.
276InFlightDiagnostic Operation::emitWarning(const Twine &message) {
277 InFlightDiagnostic diag = mlir::emitWarning(loc: getLoc(), message);
278 if (getContext()->shouldPrintOpOnDiagnostic())
279 diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
280 return diag;
281}
282
283/// Emit a remark about this operation, reporting up to any diagnostic
284/// handlers that may be listening.
285InFlightDiagnostic Operation::emitRemark(const Twine &message) {
286 InFlightDiagnostic diag = mlir::emitRemark(loc: getLoc(), message);
287 if (getContext()->shouldPrintOpOnDiagnostic())
288 diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
289 return diag;
290}
291
292DictionaryAttr Operation::getAttrDictionary() {
293 if (getPropertiesStorageSize()) {
294 NamedAttrList attrsList = attrs;
295 getName().populateInherentAttrs(op: this, attrs&: attrsList);
296 return attrsList.getDictionary(context: getContext());
297 }
298 return attrs;
299}
300
301void Operation::setAttrs(DictionaryAttr newAttrs) {
302 assert(newAttrs && "expected valid attribute dictionary");
303 if (getPropertiesStorageSize()) {
304 // We're spliting the providing DictionaryAttr by removing the inherentAttr
305 // which will be stored in the properties.
306 SmallVector<NamedAttribute> discardableAttrs;
307 discardableAttrs.reserve(N: newAttrs.size());
308 for (NamedAttribute attr : newAttrs) {
309 if (getInherentAttr(name: attr.getName()))
310 setInherentAttr(name: attr.getName(), value: attr.getValue());
311 else
312 discardableAttrs.push_back(Elt: attr);
313 }
314 if (discardableAttrs.size() != newAttrs.size())
315 newAttrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs);
316 }
317 attrs = newAttrs;
318}
319void Operation::setAttrs(ArrayRef<NamedAttribute> newAttrs) {
320 if (getPropertiesStorageSize()) {
321 // We're spliting the providing array of attributes by removing the
322 // inherentAttr which will be stored in the properties.
323 SmallVector<NamedAttribute> discardableAttrs;
324 discardableAttrs.reserve(N: newAttrs.size());
325 for (NamedAttribute attr : newAttrs) {
326 if (getInherentAttr(name: attr.getName()))
327 setInherentAttr(name: attr.getName(), value: attr.getValue());
328 else
329 discardableAttrs.push_back(Elt: attr);
330 }
331 attrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs);
332 return;
333 }
334 attrs = DictionaryAttr::get(context: getContext(), value: newAttrs);
335}
336
337std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
338 return getName().getInherentAttr(op: this, name);
339}
340
341void Operation::setInherentAttr(StringAttr name, Attribute value) {
342 getName().setInherentAttr(op: this, name, value);
343}
344
345Attribute Operation::getPropertiesAsAttribute() {
346 std::optional<RegisteredOperationName> info = getRegisteredInfo();
347 if (LLVM_UNLIKELY(!info))
348 return *getPropertiesStorage().as<Attribute *>();
349 return info->getOpPropertiesAsAttribute(op: this);
350}
351LogicalResult Operation::setPropertiesFromAttribute(
352 Attribute attr, function_ref<InFlightDiagnostic()> emitError) {
353 std::optional<RegisteredOperationName> info = getRegisteredInfo();
354 if (LLVM_UNLIKELY(!info)) {
355 *getPropertiesStorage().as<Attribute *>() = attr;
356 return success();
357 }
358 return info->setOpPropertiesFromAttribute(
359 opName: this->getName(), properties: this->getPropertiesStorage(), attr, emitError);
360}
361
362void Operation::copyProperties(PropertyRef rhs) {
363 name.copyOpProperties(lhs: getPropertiesStorage(), rhs);
364}
365
366llvm::hash_code Operation::hashProperties() {
367 return name.hashOpProperties(properties: getPropertiesStorage());
368}
369
370//===----------------------------------------------------------------------===//
371// Operation Ordering
372//===----------------------------------------------------------------------===//
373
374/// Given an operation 'other' that is within the same parent block, return
375/// whether the current operation is before 'other' in the operation list
376/// of the parent block.
377/// Note: This function has an average complexity of O(1), but worst case may
378/// take O(N) where N is the number of operations within the parent block.
379bool Operation::isBeforeInBlock(Operation *other) {
380 assert(block && "Operations without parent blocks have no order.");
381 assert(other && other->block == block &&
382 "Expected other operation to have the same parent block.");
383 // If the order of the block is already invalid, directly recompute the
384 // parent.
385 if (!block->isOpOrderValid()) {
386 block->recomputeOpOrder();
387 } else {
388 // Update the order either operation if necessary.
389 updateOrderIfNecessary();
390 other->updateOrderIfNecessary();
391 }
392
393 return orderIndex < other->orderIndex;
394}
395
396/// Update the order index of this operation of this operation if necessary,
397/// potentially recomputing the order of the parent block.
398void Operation::updateOrderIfNecessary() {
399 assert(block && "expected valid parent");
400
401 // If the order is valid for this operation there is nothing to do.
402 if (hasValidOrder() || llvm::hasSingleElement(C&: *block))
403 return;
404 Operation *blockFront = &block->front();
405 Operation *blockBack = &block->back();
406
407 // This method is expected to only be invoked on blocks with more than one
408 // operation.
409 assert(blockFront != blockBack && "expected more than one operation");
410
411 // If the operation is at the end of the block.
412 if (this == blockBack) {
413 Operation *prevNode = getPrevNode();
414 if (!prevNode->hasValidOrder())
415 return block->recomputeOpOrder();
416
417 // Add the stride to the previous operation.
418 orderIndex = prevNode->orderIndex + kOrderStride;
419 return;
420 }
421
422 // If this is the first operation try to use the next operation to compute the
423 // ordering.
424 if (this == blockFront) {
425 Operation *nextNode = getNextNode();
426 if (!nextNode->hasValidOrder())
427 return block->recomputeOpOrder();
428 // There is no order to give this operation.
429 if (nextNode->orderIndex == 0)
430 return block->recomputeOpOrder();
431
432 // If we can't use the stride, just take the middle value left. This is safe
433 // because we know there is at least one valid index to assign to.
434 if (nextNode->orderIndex <= kOrderStride)
435 orderIndex = (nextNode->orderIndex / 2);
436 else
437 orderIndex = kOrderStride;
438 return;
439 }
440
441 // Otherwise, this operation is between two others. Place this operation in
442 // the middle of the previous and next if possible.
443 Operation *prevNode = getPrevNode(), *nextNode = getNextNode();
444 if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder())
445 return block->recomputeOpOrder();
446 unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex;
447
448 // Check to see if there is a valid order between the two.
449 if (prevOrder + 1 == nextOrder)
450 return block->recomputeOpOrder();
451 orderIndex = prevOrder + ((nextOrder - prevOrder) / 2);
452}
453
454//===----------------------------------------------------------------------===//
455// ilist_traits for Operation
456//===----------------------------------------------------------------------===//
457
458auto llvm::ilist_detail::SpecificNodeAccess<
459 llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>::
460 getNodePtr(pointer n) -> node_type * {
461 return NodeAccess::getNodePtr<OptionsT>(N: n);
462}
463
464auto llvm::ilist_detail::SpecificNodeAccess<
465 llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>::
466 getNodePtr(const_pointer n) -> const node_type * {
467 return NodeAccess::getNodePtr<OptionsT>(N: n);
468}
469
470auto llvm::ilist_detail::SpecificNodeAccess<
471 llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>::
472 getValuePtr(node_type *n) -> pointer {
473 return NodeAccess::getValuePtr<OptionsT>(N: n);
474}
475
476auto llvm::ilist_detail::SpecificNodeAccess<
477 llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>::
478 getValuePtr(const node_type *n) -> const_pointer {
479 return NodeAccess::getValuePtr<OptionsT>(N: n);
480}
481
482void llvm::ilist_traits<::mlir::Operation>::deleteNode(Operation *op) {
483 op->destroy();
484}
485
486Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() {
487 size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr))));
488 iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this));
489 return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset);
490}
491
492/// This is a trait method invoked when an operation is added to a block. We
493/// keep the block pointer up to date.
494void llvm::ilist_traits<::mlir::Operation>::addNodeToList(Operation *op) {
495 assert(!op->getBlock() && "already in an operation block!");
496 op->block = getContainingBlock();
497
498 // Invalidate the order on the operation.
499 op->orderIndex = Operation::kInvalidOrderIdx;
500}
501
502/// This is a trait method invoked when an operation is removed from a block.
503/// We keep the block pointer up to date.
504void llvm::ilist_traits<::mlir::Operation>::removeNodeFromList(Operation *op) {
505 assert(op->block && "not already in an operation block!");
506 op->block = nullptr;
507}
508
509/// This is a trait method invoked when an operation is moved from one block
510/// to another. We keep the block pointer up to date.
511void llvm::ilist_traits<::mlir::Operation>::transferNodesFromList(
512 ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) {
513 Block *curParent = getContainingBlock();
514
515 // Invalidate the ordering of the parent block.
516 curParent->invalidateOpOrder();
517
518 // If we are transferring operations within the same block, the block
519 // pointer doesn't need to be updated.
520 if (curParent == otherList.getContainingBlock())
521 return;
522
523 // Update the 'block' member of each operation.
524 for (; first != last; ++first)
525 first->block = curParent;
526}
527
528/// Remove this operation (and its descendants) from its Block and delete
529/// all of them.
530void Operation::erase() {
531 if (auto *parent = getBlock())
532 parent->getOperations().erase(IT: this);
533 else
534 destroy();
535}
536
537/// Remove the operation from its parent block, but don't delete it.
538void Operation::remove() {
539 if (Block *parent = getBlock())
540 parent->getOperations().remove(IT: this);
541}
542
543/// Unlink this operation from its current block and insert it right before
544/// `existingOp` which may be in the same or another block in the same
545/// function.
546void Operation::moveBefore(Operation *existingOp) {
547 moveBefore(block: existingOp->getBlock(), iterator: existingOp->getIterator());
548}
549
550/// Unlink this operation from its current basic block and insert it right
551/// before `iterator` in the specified basic block.
552void Operation::moveBefore(Block *block,
553 llvm::iplist<Operation>::iterator iterator) {
554 assert(getBlock() &&
555 "cannot move an operation that isn't contained in a block");
556 block->getOperations().splice(where: iterator, L2&: getBlock()->getOperations(),
557 first: getIterator());
558}
559
560/// Unlink this operation from its current block and insert it right after
561/// `existingOp` which may be in the same or another block in the same function.
562void Operation::moveAfter(Operation *existingOp) {
563 moveAfter(block: existingOp->getBlock(), iterator: existingOp->getIterator());
564}
565
566/// Unlink this operation from its current block and insert it right after
567/// `iterator` in the specified block.
568void Operation::moveAfter(Block *block,
569 llvm::iplist<Operation>::iterator iterator) {
570 assert(iterator != block->end() && "cannot move after end of block");
571 moveBefore(block, iterator: std::next(x: iterator));
572}
573
574/// This drops all operand uses from this operation, which is an essential
575/// step in breaking cyclic dependences between references when they are to
576/// be deleted.
577void Operation::dropAllReferences() {
578 for (auto &op : getOpOperands())
579 op.drop();
580
581 for (auto &region : getRegions())
582 region.dropAllReferences();
583
584 for (auto &dest : getBlockOperands())
585 dest.drop();
586}
587
588/// This drops all uses of any values defined by this operation or its nested
589/// regions, wherever they are located.
590void Operation::dropAllDefinedValueUses() {
591 dropAllUses();
592
593 for (auto &region : getRegions())
594 for (auto &block : region)
595 block.dropAllDefinedValueUses();
596}
597
598void Operation::setSuccessor(Block *block, unsigned index) {
599 assert(index < getNumSuccessors());
600 getBlockOperands()[index].set(block);
601}
602
603#ifndef NDEBUG
604/// Assert that the folded results (in case of values) have the same type as
605/// the results of the given op.
606static void checkFoldResultTypes(Operation *op,
607 SmallVectorImpl<OpFoldResult> &results) {
608 if (results.empty())
609 return;
610
611 for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) {
612 if (auto value = dyn_cast<Value>(ofr)) {
613 if (value.getType() != opResult.getType()) {
614 op->emitOpError() << "folder produced a value of incorrect type: "
615 << value.getType()
616 << ", expected: " << opResult.getType();
617 assert(false && "incorrect fold result type");
618 }
619 }
620 }
621}
622#endif // NDEBUG
623
624/// Attempt to fold this operation using the Op's registered foldHook.
625LogicalResult Operation::fold(ArrayRef<Attribute> operands,
626 SmallVectorImpl<OpFoldResult> &results) {
627 // If we have a registered operation definition matching this one, use it to
628 // try to constant fold the operation.
629 if (succeeded(Result: name.foldHook(op: this, operands, results))) {
630#ifndef NDEBUG
631 checkFoldResultTypes(this, results);
632#endif // NDEBUG
633 return success();
634 }
635
636 // Otherwise, fall back on the dialect hook to handle it.
637 Dialect *dialect = getDialect();
638 if (!dialect)
639 return failure();
640
641 auto *interface = dyn_cast<DialectFoldInterface>(Val: dialect);
642 if (!interface)
643 return failure();
644
645 LogicalResult status = interface->fold(op: this, operands, results);
646#ifndef NDEBUG
647 if (succeeded(status))
648 checkFoldResultTypes(this, results);
649#endif // NDEBUG
650 return status;
651}
652
653LogicalResult Operation::fold(SmallVectorImpl<OpFoldResult> &results) {
654 // Check if any operands are constants.
655 SmallVector<Attribute> constants;
656 constants.assign(NumElts: getNumOperands(), Elt: Attribute());
657 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
658 matchPattern(value: getOperand(idx: i), pattern: m_Constant(bind_value: &constants[i]));
659 return fold(operands: constants, results);
660}
661
662/// Emit an error with the op name prefixed, like "'dim' op " which is
663/// convenient for verifiers.
664InFlightDiagnostic Operation::emitOpError(const Twine &message) {
665 return emitError() << "'" << getName() << "' op " << message;
666}
667
668//===----------------------------------------------------------------------===//
669// Operation Cloning
670//===----------------------------------------------------------------------===//
671
672Operation::CloneOptions::CloneOptions()
673 : cloneRegionsFlag(false), cloneOperandsFlag(false),
674 resultTypes(std::nullopt) {}
675
676Operation::CloneOptions::CloneOptions(
677 bool cloneRegions, bool cloneOperands,
678 std::optional<SmallVector<Type>> resultTypes)
679 : cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands),
680 resultTypes(std::move(resultTypes)) {}
681
682Operation::CloneOptions Operation::CloneOptions::all() {
683 return CloneOptions().cloneRegions().cloneOperands().withResultTypes(
684 resultTypes: std::nullopt);
685}
686
687Operation::CloneOptions &Operation::CloneOptions::cloneRegions(bool enable) {
688 cloneRegionsFlag = enable;
689 return *this;
690}
691
692Operation::CloneOptions &Operation::CloneOptions::cloneOperands(bool enable) {
693 cloneOperandsFlag = enable;
694 return *this;
695}
696
697Operation::CloneOptions &Operation::CloneOptions::withResultTypes(
698 std::optional<SmallVector<Type>> resultTypes) {
699 this->resultTypes = std::move(resultTypes);
700 return *this;
701}
702
703/// Create a deep copy of this operation but keep the operation regions empty.
704/// Operands are remapped using `mapper` (if present), and `mapper` is updated
705/// to contain the results. The `mapResults` flag specifies whether the results
706/// of the cloned operation should be added to the map.
707Operation *Operation::cloneWithoutRegions(IRMapping &mapper) {
708 return clone(mapper, options: CloneOptions::all().cloneRegions(enable: false));
709}
710
711Operation *Operation::cloneWithoutRegions() {
712 IRMapping mapper;
713 return cloneWithoutRegions(mapper);
714}
715
716/// Create a deep copy of this operation, remapping any operands that use
717/// values outside of the operation using the map that is provided (leaving
718/// them alone if no entry is present). Replaces references to cloned
719/// sub-operations to the corresponding operation that is copied, and adds
720/// those mappings to the map.
721Operation *Operation::clone(IRMapping &mapper, const CloneOptions &options) {
722 SmallVector<Value, 8> operands;
723 SmallVector<Block *, 2> successors;
724
725 // Remap the operands.
726 if (options.shouldCloneOperands()) {
727 operands.reserve(N: getNumOperands());
728 for (auto opValue : getOperands())
729 operands.push_back(Elt: mapper.lookupOrDefault(from: opValue));
730 }
731
732 // Remap the successors.
733 successors.reserve(N: getNumSuccessors());
734 for (Block *successor : getSuccessors())
735 successors.push_back(Elt: mapper.lookupOrDefault(from: successor));
736
737 // Create the new operation.
738 auto *newOp = create(location: getLoc(), name: getName(),
739 resultTypes: options.resultTypesOr(defaultResultTypes: getResultTypes()), operands, attributes: attrs,
740 properties: getPropertiesStorage(), successors, numRegions: getNumRegions());
741 mapper.map(from: this, to: newOp);
742
743 // Clone the regions.
744 if (options.shouldCloneRegions()) {
745 for (unsigned i = 0; i != numRegions; ++i)
746 getRegion(index: i).cloneInto(dest: &newOp->getRegion(index: i), mapper);
747 }
748
749 // Remember the mapping of any results.
750 if (options.shouldCloneResults())
751 for (unsigned i = 0, e = getNumResults(); i != e; ++i)
752 mapper.map(from: getResult(idx: i), to: newOp->getResult(idx: i));
753
754 return newOp;
755}
756
757Operation *Operation::clone(const CloneOptions &options) {
758 IRMapping mapper;
759 return clone(mapper, options);
760}
761
762//===----------------------------------------------------------------------===//
763// OpState trait class.
764//===----------------------------------------------------------------------===//
765
766// The fallback for the parser is to try for a dialect operation parser.
767// Otherwise, reject the custom assembly form.
768ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) {
769 if (auto parseFn = result.name.getDialect()->getParseOperationHook(
770 opName: result.name.getStringRef()))
771 return (*parseFn)(parser, result);
772 return parser.emitError(loc: parser.getNameLoc(), message: "has no custom assembly form");
773}
774
775// The fallback for the printer is to try for a dialect operation printer.
776// Otherwise, it prints the generic form.
777void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
778 if (auto printFn = op->getDialect()->getOperationPrinter(op)) {
779 printOpName(op, p, defaultDialect);
780 printFn(op, p);
781 } else {
782 p.printGenericOp(op);
783 }
784}
785
786#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
787void OpState::dump() { state->dump(); }
788#endif
789
790/// Print an operation name, eliding the dialect prefix if necessary and doesn't
791/// lead to ambiguities.
792void OpState::printOpName(Operation *op, OpAsmPrinter &p,
793 StringRef defaultDialect) {
794 StringRef name = op->getName().getStringRef();
795 if (name.starts_with(Prefix: (defaultDialect + ".").str()) && name.count(C: '.') == 1)
796 name = name.drop_front(N: defaultDialect.size() + 1);
797 p.getStream() << name;
798}
799
800/// Parse properties as a Attribute.
801ParseResult OpState::genericParseProperties(OpAsmParser &parser,
802 Attribute &result) {
803 if (succeeded(Result: parser.parseOptionalLess())) { // The less is optional.
804 if (parser.parseAttribute(result) || parser.parseGreater())
805 return failure();
806 }
807 return success();
808}
809
810/// Print the properties as a Attribute with names not included within
811/// 'elidedProps'. A leading space is printed as part of the properties, so
812/// that nothing is emitted at all when every property is elided.
813void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
814 ArrayRef<StringRef> elidedProps) {
815 if (!properties)
816 return;
817 auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(Val&: properties);
818 if (dictAttr && !elidedProps.empty()) {
819 ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
820 llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
821 elidedProps.end());
822 auto filteredAttrs =
823 llvm::make_filter_range(Range&: attrs, Pred: [&](NamedAttribute attr) {
824 return !elidedAttrsSet.contains(V: attr.getName().strref());
825 });
826 if (!filteredAttrs.empty()) {
827 p << " <{";
828 interleaveComma(c: filteredAttrs, os&: p, each_fn: [&](NamedAttribute attr) {
829 p.printNamedAttribute(attr);
830 });
831 p << "}>";
832 }
833 } else {
834 p << " <" << properties << ">";
835 }
836}
837
838/// Emit an error about fatal conditions with this operation, reporting up to
839/// any diagnostic handlers that may be listening.
840InFlightDiagnostic OpState::emitError(const Twine &message) {
841 return getOperation()->emitError(message);
842}
843
844/// Emit an error with the op name prefixed, like "'dim' op " which is
845/// convenient for verifiers.
846InFlightDiagnostic OpState::emitOpError(const Twine &message) {
847 return getOperation()->emitOpError(message);
848}
849
850/// Emit a warning about this operation, reporting up to any diagnostic
851/// handlers that may be listening.
852InFlightDiagnostic OpState::emitWarning(const Twine &message) {
853 return getOperation()->emitWarning(message);
854}
855
856/// Emit a remark about this operation, reporting up to any diagnostic
857/// handlers that may be listening.
858InFlightDiagnostic OpState::emitRemark(const Twine &message) {
859 return getOperation()->emitRemark(message);
860}
861
862//===----------------------------------------------------------------------===//
863// Op Trait implementations
864//===----------------------------------------------------------------------===//
865
866LogicalResult
867OpTrait::impl::foldCommutative(Operation *op, ArrayRef<Attribute> operands,
868 SmallVectorImpl<OpFoldResult> &results) {
869 // Nothing to fold if there are not at least 2 operands.
870 if (op->getNumOperands() < 2)
871 return failure();
872 // Move all constant operands to the end.
873 OpOperand *operandsBegin = op->getOpOperands().begin();
874 auto isNonConstant = [&](OpOperand &o) {
875 return !static_cast<bool>(operands[std::distance(first: operandsBegin, last: &o)]);
876 };
877 auto *firstConstantIt = llvm::find_if_not(Range: op->getOpOperands(), P: isNonConstant);
878 auto *newConstantIt = std::stable_partition(
879 first: firstConstantIt, last: op->getOpOperands().end(), pred: isNonConstant);
880 // Return success if the op was modified.
881 return success(IsSuccess: firstConstantIt != newConstantIt);
882}
883
884OpFoldResult OpTrait::impl::foldIdempotent(Operation *op) {
885 if (op->getNumOperands() == 1) {
886 auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
887 if (argumentOp && op->getName() == argumentOp->getName()) {
888 // Replace the outer operation output with the inner operation.
889 return op->getOperand(idx: 0);
890 }
891 } else if (op->getOperand(idx: 0) == op->getOperand(idx: 1)) {
892 return op->getOperand(idx: 0);
893 }
894
895 return {};
896}
897
898OpFoldResult OpTrait::impl::foldInvolution(Operation *op) {
899 auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
900 if (argumentOp && op->getName() == argumentOp->getName()) {
901 // Replace the outer involutions output with inner's input.
902 return argumentOp->getOperand(idx: 0);
903 }
904
905 return {};
906}
907
908LogicalResult OpTrait::impl::verifyZeroOperands(Operation *op) {
909 if (op->getNumOperands() != 0)
910 return op->emitOpError() << "requires zero operands";
911 return success();
912}
913
914LogicalResult OpTrait::impl::verifyOneOperand(Operation *op) {
915 if (op->getNumOperands() != 1)
916 return op->emitOpError() << "requires a single operand";
917 return success();
918}
919
920LogicalResult OpTrait::impl::verifyNOperands(Operation *op,
921 unsigned numOperands) {
922 if (op->getNumOperands() != numOperands) {
923 return op->emitOpError() << "expected " << numOperands
924 << " operands, but found " << op->getNumOperands();
925 }
926 return success();
927}
928
929LogicalResult OpTrait::impl::verifyAtLeastNOperands(Operation *op,
930 unsigned numOperands) {
931 if (op->getNumOperands() < numOperands)
932 return op->emitOpError()
933 << "expected " << numOperands << " or more operands, but found "
934 << op->getNumOperands();
935 return success();
936}
937
938/// If this is a vector type, or a tensor type, return the scalar element type
939/// that it is built around, otherwise return the type unmodified.
940static Type getTensorOrVectorElementType(Type type) {
941 if (auto vec = llvm::dyn_cast<VectorType>(Val&: type))
942 return vec.getElementType();
943
944 // Look through tensor<vector<...>> to find the underlying element type.
945 if (auto tensor = llvm::dyn_cast<TensorType>(Val&: type))
946 return getTensorOrVectorElementType(type: tensor.getElementType());
947 return type;
948}
949
950LogicalResult OpTrait::impl::verifyIsIdempotent(Operation *op) {
951 // FIXME: Add back check for no side effects on operation.
952 // Currently adding it would cause the shared library build
953 // to fail since there would be a dependency of IR on SideEffectInterfaces
954 // which is cyclical.
955 return success();
956}
957
958LogicalResult OpTrait::impl::verifyIsInvolution(Operation *op) {
959 // FIXME: Add back check for no side effects on operation.
960 // Currently adding it would cause the shared library build
961 // to fail since there would be a dependency of IR on SideEffectInterfaces
962 // which is cyclical.
963 return success();
964}
965
966LogicalResult
967OpTrait::impl::verifyOperandsAreSignlessIntegerLike(Operation *op) {
968 for (auto opType : op->getOperandTypes()) {
969 auto type = getTensorOrVectorElementType(type: opType);
970 if (!type.isSignlessIntOrIndex())
971 return op->emitOpError() << "requires an integer or index type";
972 }
973 return success();
974}
975
976LogicalResult OpTrait::impl::verifyOperandsAreFloatLike(Operation *op) {
977 for (auto opType : op->getOperandTypes()) {
978 auto type = getTensorOrVectorElementType(type: opType);
979 if (!llvm::isa<FloatType>(Val: type))
980 return op->emitOpError(message: "requires a float type");
981 }
982 return success();
983}
984
985LogicalResult OpTrait::impl::verifySameTypeOperands(Operation *op) {
986 // Zero or one operand always have the "same" type.
987 unsigned nOperands = op->getNumOperands();
988 if (nOperands < 2)
989 return success();
990
991 auto type = op->getOperand(idx: 0).getType();
992 for (auto opType : llvm::drop_begin(RangeOrContainer: op->getOperandTypes(), N: 1))
993 if (opType != type)
994 return op->emitOpError() << "requires all operands to have the same type";
995 return success();
996}
997
998LogicalResult OpTrait::impl::verifyZeroRegions(Operation *op) {
999 if (op->getNumRegions() != 0)
1000 return op->emitOpError() << "requires zero regions";
1001 return success();
1002}
1003
1004LogicalResult OpTrait::impl::verifyOneRegion(Operation *op) {
1005 if (op->getNumRegions() != 1)
1006 return op->emitOpError() << "requires one region";
1007 return success();
1008}
1009
1010LogicalResult OpTrait::impl::verifyNRegions(Operation *op,
1011 unsigned numRegions) {
1012 if (op->getNumRegions() != numRegions)
1013 return op->emitOpError() << "expected " << numRegions << " regions";
1014 return success();
1015}
1016
1017LogicalResult OpTrait::impl::verifyAtLeastNRegions(Operation *op,
1018 unsigned numRegions) {
1019 if (op->getNumRegions() < numRegions)
1020 return op->emitOpError() << "expected " << numRegions << " or more regions";
1021 return success();
1022}
1023
1024LogicalResult OpTrait::impl::verifyZeroResults(Operation *op) {
1025 if (op->getNumResults() != 0)
1026 return op->emitOpError() << "requires zero results";
1027 return success();
1028}
1029
1030LogicalResult OpTrait::impl::verifyOneResult(Operation *op) {
1031 if (op->getNumResults() != 1)
1032 return op->emitOpError() << "requires one result";
1033 return success();
1034}
1035
1036LogicalResult OpTrait::impl::verifyNResults(Operation *op,
1037 unsigned numOperands) {
1038 if (op->getNumResults() != numOperands)
1039 return op->emitOpError() << "expected " << numOperands << " results";
1040 return success();
1041}
1042
1043LogicalResult OpTrait::impl::verifyAtLeastNResults(Operation *op,
1044 unsigned numOperands) {
1045 if (op->getNumResults() < numOperands)
1046 return op->emitOpError()
1047 << "expected " << numOperands << " or more results";
1048 return success();
1049}
1050
1051LogicalResult OpTrait::impl::verifySameOperandsShape(Operation *op) {
1052 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1053 return failure();
1054
1055 if (failed(Result: verifyCompatibleShapes(types: op->getOperandTypes())))
1056 return op->emitOpError() << "requires the same shape for all operands";
1057
1058 return success();
1059}
1060
1061LogicalResult OpTrait::impl::verifySameOperandsAndResultShape(Operation *op) {
1062 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1063 failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1064 return failure();
1065
1066 SmallVector<Type, 8> types(op->getOperandTypes());
1067 types.append(RHS: llvm::to_vector<4>(Range: op->getResultTypes()));
1068
1069 if (failed(Result: verifyCompatibleShapes(types)))
1070 return op->emitOpError()
1071 << "requires the same shape for all operands and results";
1072
1073 return success();
1074}
1075
1076LogicalResult OpTrait::impl::verifySameOperandsElementType(Operation *op) {
1077 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1078 return failure();
1079 auto elementType = getElementTypeOrSelf(val: op->getOperand(idx: 0));
1080
1081 for (auto operand : llvm::drop_begin(RangeOrContainer: op->getOperands(), N: 1)) {
1082 if (getElementTypeOrSelf(val: operand) != elementType)
1083 return op->emitOpError(message: "requires the same element type for all operands");
1084 }
1085
1086 return success();
1087}
1088
1089LogicalResult
1090OpTrait::impl::verifySameOperandsAndResultElementType(Operation *op) {
1091 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1092 failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1093 return failure();
1094
1095 auto elementType = getElementTypeOrSelf(val: op->getResult(idx: 0));
1096
1097 // Verify result element type matches first result's element type.
1098 for (auto result : llvm::drop_begin(RangeOrContainer: op->getResults(), N: 1)) {
1099 if (getElementTypeOrSelf(val: result) != elementType)
1100 return op->emitOpError(
1101 message: "requires the same element type for all operands and results");
1102 }
1103
1104 // Verify operand's element type matches first result's element type.
1105 for (auto operand : op->getOperands()) {
1106 if (getElementTypeOrSelf(val: operand) != elementType)
1107 return op->emitOpError(
1108 message: "requires the same element type for all operands and results");
1109 }
1110
1111 return success();
1112}
1113
1114LogicalResult OpTrait::impl::verifySameOperandsAndResultType(Operation *op) {
1115 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1116 failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1117 return failure();
1118
1119 auto type = op->getResult(idx: 0).getType();
1120 auto elementType = getElementTypeOrSelf(type);
1121 Attribute encoding = nullptr;
1122 if (auto rankedType = dyn_cast<RankedTensorType>(Val&: type))
1123 encoding = rankedType.getEncoding();
1124 for (auto resultType : llvm::drop_begin(RangeOrContainer: op->getResultTypes())) {
1125 if (getElementTypeOrSelf(type: resultType) != elementType ||
1126 failed(Result: verifyCompatibleShape(type1: resultType, type2: type)))
1127 return op->emitOpError()
1128 << "requires the same type for all operands and results";
1129 if (encoding)
1130 if (auto rankedType = dyn_cast<RankedTensorType>(Val&: resultType);
1131 encoding != rankedType.getEncoding())
1132 return op->emitOpError()
1133 << "requires the same encoding for all operands and results";
1134 }
1135 for (auto opType : op->getOperandTypes()) {
1136 if (getElementTypeOrSelf(type: opType) != elementType ||
1137 failed(Result: verifyCompatibleShape(type1: opType, type2: type)))
1138 return op->emitOpError()
1139 << "requires the same type for all operands and results";
1140 if (encoding)
1141 if (auto rankedType = dyn_cast<RankedTensorType>(Val&: opType);
1142 encoding != rankedType.getEncoding())
1143 return op->emitOpError()
1144 << "requires the same encoding for all operands and results";
1145 }
1146 return success();
1147}
1148
1149LogicalResult OpTrait::impl::verifySameOperandsAndResultRank(Operation *op) {
1150 if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1151 return failure();
1152
1153 // delegate function that returns true if type is a shaped type with known
1154 // rank
1155 auto hasRank = [](const Type type) {
1156 if (auto shapedType = dyn_cast<ShapedType>(Val: type))
1157 return shapedType.hasRank();
1158
1159 return false;
1160 };
1161
1162 auto rankedOperandTypes =
1163 llvm::make_filter_range(Range: op->getOperandTypes(), Pred: hasRank);
1164 auto rankedResultTypes =
1165 llvm::make_filter_range(Range: op->getResultTypes(), Pred: hasRank);
1166
1167 // If all operands and results are unranked, then no further verification.
1168 if (rankedOperandTypes.empty() && rankedResultTypes.empty())
1169 return success();
1170
1171 // delegate function that returns rank of shaped type with known rank
1172 auto getRank = [](const Type type) {
1173 return cast<ShapedType>(Val: type).getRank();
1174 };
1175
1176 auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
1177 : getRank(*rankedResultTypes.begin());
1178
1179 for (const auto type : rankedOperandTypes) {
1180 if (rank != getRank(type)) {
1181 return op->emitOpError(message: "operands don't have matching ranks");
1182 }
1183 }
1184
1185 for (const auto type : rankedResultTypes) {
1186 if (rank != getRank(type)) {
1187 return op->emitOpError(message: "result type has different rank than operands");
1188 }
1189 }
1190
1191 return success();
1192}
1193
1194LogicalResult OpTrait::impl::verifyIsTerminator(Operation *op) {
1195 Block *block = op->getBlock();
1196 // Verify that the operation is at the end of the respective parent block.
1197 if (!block || &block->back() != op)
1198 return op->emitOpError(message: "must be the last operation in the parent block");
1199 return success();
1200}
1201
1202static LogicalResult verifyTerminatorSuccessors(Operation *op) {
1203 auto *parent = op->getParentRegion();
1204
1205 // Verify that the operands lines up with the BB arguments in the successor.
1206 for (Block *succ : op->getSuccessors())
1207 if (succ->getParent() != parent)
1208 return op->emitError(message: "reference to block defined in another region");
1209 return success();
1210}
1211
1212LogicalResult OpTrait::impl::verifyZeroSuccessors(Operation *op) {
1213 if (op->getNumSuccessors() != 0) {
1214 return op->emitOpError(message: "requires 0 successors but found ")
1215 << op->getNumSuccessors();
1216 }
1217 return success();
1218}
1219
1220LogicalResult OpTrait::impl::verifyOneSuccessor(Operation *op) {
1221 if (op->getNumSuccessors() != 1) {
1222 return op->emitOpError(message: "requires 1 successor but found ")
1223 << op->getNumSuccessors();
1224 }
1225 return verifyTerminatorSuccessors(op);
1226}
1227LogicalResult OpTrait::impl::verifyNSuccessors(Operation *op,
1228 unsigned numSuccessors) {
1229 if (op->getNumSuccessors() != numSuccessors) {
1230 return op->emitOpError(message: "requires ")
1231 << numSuccessors << " successors but found "
1232 << op->getNumSuccessors();
1233 }
1234 return verifyTerminatorSuccessors(op);
1235}
1236LogicalResult OpTrait::impl::verifyAtLeastNSuccessors(Operation *op,
1237 unsigned numSuccessors) {
1238 if (op->getNumSuccessors() < numSuccessors) {
1239 return op->emitOpError(message: "requires at least ")
1240 << numSuccessors << " successors but found "
1241 << op->getNumSuccessors();
1242 }
1243 return verifyTerminatorSuccessors(op);
1244}
1245
1246LogicalResult OpTrait::impl::verifyResultsAreBoolLike(Operation *op) {
1247 for (auto resultType : op->getResultTypes()) {
1248 auto elementType = getTensorOrVectorElementType(type: resultType);
1249 bool isBoolType = elementType.isInteger(width: 1);
1250 if (!isBoolType)
1251 return op->emitOpError() << "requires a bool result type";
1252 }
1253
1254 return success();
1255}
1256
1257LogicalResult OpTrait::impl::verifyResultsAreFloatLike(Operation *op) {
1258 for (auto resultType : op->getResultTypes())
1259 if (!llvm::isa<FloatType>(Val: getTensorOrVectorElementType(type: resultType)))
1260 return op->emitOpError() << "requires a floating point type";
1261
1262 return success();
1263}
1264
1265LogicalResult
1266OpTrait::impl::verifyResultsAreSignlessIntegerLike(Operation *op) {
1267 for (auto resultType : op->getResultTypes())
1268 if (!getTensorOrVectorElementType(type: resultType).isSignlessIntOrIndex())
1269 return op->emitOpError() << "requires an integer or index type";
1270 return success();
1271}
1272
1273LogicalResult OpTrait::impl::verifyValueSizeAttr(Operation *op,
1274 StringRef attrName,
1275 StringRef valueGroupName,
1276 size_t expectedCount) {
1277 auto sizeAttr = dyn_cast_or_null<DenseI32ArrayAttr>(
1278 Val: op->getInherentAttr(name: attrName).value_or(u: Attribute{}));
1279 if (!sizeAttr)
1280 return op->emitOpError(message: "requires dense i32 array attribute '")
1281 << attrName << "'";
1282
1283 ArrayRef<int32_t> sizes = sizeAttr.asArrayRef();
1284 if (llvm::any_of(Range&: sizes, P: [](int32_t element) { return element < 0; }))
1285 return op->emitOpError(message: "'")
1286 << attrName << "' attribute cannot have negative elements";
1287
1288 size_t totalCount = llvm::sum_of(Range&: sizes, Init: size_t(0));
1289 if (totalCount != expectedCount)
1290 return op->emitOpError()
1291 << valueGroupName << " count (" << expectedCount
1292 << ") does not match with the total size (" << totalCount
1293 << ") specified in attribute '" << attrName << "'";
1294 return success();
1295}
1296
1297LogicalResult OpTrait::impl::verifyOperandSizeAttr(Operation *op,
1298 StringRef attrName) {
1299 return verifyValueSizeAttr(op, attrName, valueGroupName: "operand", expectedCount: op->getNumOperands());
1300}
1301
1302LogicalResult OpTrait::impl::verifyResultSizeAttr(Operation *op,
1303 StringRef attrName) {
1304 return verifyValueSizeAttr(op, attrName, valueGroupName: "result", expectedCount: op->getNumResults());
1305}
1306
1307LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) {
1308 for (Region &region : op->getRegions()) {
1309 if (region.empty())
1310 continue;
1311
1312 if (region.getNumArguments() != 0) {
1313 if (op->getNumRegions() > 1)
1314 return op->emitOpError(message: "region #")
1315 << region.getRegionNumber() << " should have no arguments";
1316 return op->emitOpError(message: "region should have no arguments");
1317 }
1318 }
1319 return success();
1320}
1321
1322LogicalResult OpTrait::impl::verifyElementwise(Operation *op) {
1323 auto isMappableType = llvm::IsaPred<VectorType, TensorType>;
1324 auto resultMappableTypes =
1325 llvm::filter_to_vector<1>(C: op->getResultTypes(), Pred&: isMappableType);
1326 auto operandMappableTypes =
1327 llvm::filter_to_vector<2>(C: op->getOperandTypes(), Pred&: isMappableType);
1328
1329 // If the op only has scalar operand/result types, then we have nothing to
1330 // check.
1331 if (resultMappableTypes.empty() && operandMappableTypes.empty())
1332 return success();
1333
1334 if (!resultMappableTypes.empty() && operandMappableTypes.empty())
1335 return op->emitOpError(message: "if a result is non-scalar, then at least one "
1336 "operand must be non-scalar");
1337
1338 assert(!operandMappableTypes.empty());
1339
1340 if (resultMappableTypes.empty())
1341 return op->emitOpError(message: "if an operand is non-scalar, then there must be at "
1342 "least one non-scalar result");
1343
1344 if (resultMappableTypes.size() != op->getNumResults())
1345 return op->emitOpError(
1346 message: "if an operand is non-scalar, then all results must be non-scalar");
1347
1348 SmallVector<Type, 4> types = llvm::to_vector<2>(
1349 Range: llvm::concat<Type>(Ranges&: operandMappableTypes, Ranges&: resultMappableTypes));
1350 TypeID expectedBaseTy = types.front().getTypeID();
1351 if (!llvm::all_of(Range&: types,
1352 P: [&](Type t) { return t.getTypeID() == expectedBaseTy; }) ||
1353 failed(Result: verifyCompatibleShapes(types))) {
1354 return op->emitOpError() << "all non-scalar operands/results must have the "
1355 "same shape and base type";
1356 }
1357
1358 return success();
1359}
1360
1361/// Check for any values used by operations regions attached to the
1362/// specified "IsIsolatedFromAbove" operation defined outside of it.
1363LogicalResult OpTrait::impl::verifyIsIsolatedFromAbove(Operation *isolatedOp) {
1364 assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
1365 "Intended to check IsolatedFromAbove ops");
1366
1367 // List of regions to analyze. Each region is processed independently, with
1368 // respect to the common `limit` region, so we can look at them in any order.
1369 // Therefore, use a simple vector and push/pop back the current region.
1370 SmallVector<Region *, 8> pendingRegions;
1371 for (auto &region : isolatedOp->getRegions()) {
1372 pendingRegions.push_back(Elt: &region);
1373
1374 // Traverse all operations in the region.
1375 while (!pendingRegions.empty()) {
1376 for (Operation &op : pendingRegions.pop_back_val()->getOps()) {
1377 for (Value operand : op.getOperands()) {
1378 // Check that any value that is used by an operation is defined in the
1379 // same region as either an operation result.
1380 auto *operandRegion = operand.getParentRegion();
1381 if (!operandRegion)
1382 return op.emitError(message: "operation's operand is unlinked");
1383 if (!region.isAncestor(other: operandRegion)) {
1384 return op.emitOpError(message: "using value defined outside the region")
1385 .attachNote(noteLoc: isolatedOp->getLoc())
1386 << "required by region isolation constraints";
1387 }
1388 }
1389
1390 // Schedule any regions in the operation for further checking. Don't
1391 // recurse into other IsolatedFromAbove ops, because they will check
1392 // themselves.
1393 if (op.getNumRegions() &&
1394 !op.hasTrait<OpTrait::IsIsolatedFromAbove>()) {
1395 for (Region &subRegion : op.getRegions())
1396 pendingRegions.push_back(Elt: &subRegion);
1397 }
1398 }
1399 }
1400 }
1401
1402 return success();
1403}
1404
1405bool OpTrait::hasElementwiseMappableTraits(Operation *op) {
1406 return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() &&
1407 op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>();
1408}
1409
1410//===----------------------------------------------------------------------===//
1411// Misc. utils
1412//===----------------------------------------------------------------------===//
1413
1414/// Insert an operation, generated by `buildTerminatorOp`, at the end of the
1415/// region's only block if it does not have a terminator already. If the region
1416/// is empty, insert a new block first. `buildTerminatorOp` should return the
1417/// terminator operation to insert.
1418void impl::ensureRegionTerminator(
1419 Region &region, OpBuilder &builder, Location loc,
1420 function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1421 OpBuilder::InsertionGuard guard(builder);
1422 if (region.empty())
1423 builder.createBlock(parent: &region);
1424
1425 Block &block = region.back();
1426 if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
1427 return;
1428
1429 builder.setInsertionPointToEnd(&block);
1430 builder.insert(op: buildTerminatorOp(builder, loc));
1431}
1432
1433/// Create a simple OpBuilder and forward to the OpBuilder version of this
1434/// function.
1435void impl::ensureRegionTerminator(
1436 Region &region, Builder &builder, Location loc,
1437 function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1438 OpBuilder opBuilder(builder.getContext());
1439 ensureRegionTerminator(region, builder&: opBuilder, loc, buildTerminatorOp);
1440}
1441