1//===- RewriterGen.cpp - MLIR pattern rewriter generator ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// RewriterGen uses pattern rewrite definitions to generate rewriter matchers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Support/IndentedOstream.h"
14#include "mlir/TableGen/Argument.h"
15#include "mlir/TableGen/Attribute.h"
16#include "mlir/TableGen/CodeGenHelpers.h"
17#include "mlir/TableGen/Format.h"
18#include "mlir/TableGen/GenInfo.h"
19#include "mlir/TableGen/Operator.h"
20#include "mlir/TableGen/Pattern.h"
21#include "mlir/TableGen/Predicate.h"
22#include "mlir/TableGen/Property.h"
23#include "mlir/TableGen/Type.h"
24#include "llvm/ADT/FunctionExtras.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/ADT/StringSet.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/FormatAdapters.h"
31#include "llvm/Support/PrettyStackTrace.h"
32#include "llvm/Support/Signals.h"
33#include "llvm/TableGen/Error.h"
34#include "llvm/TableGen/Main.h"
35#include "llvm/TableGen/Record.h"
36#include "llvm/TableGen/TableGenBackend.h"
37
38using namespace mlir;
39using namespace mlir::tblgen;
40
41using llvm::formatv;
42using llvm::Record;
43using llvm::RecordKeeper;
44
45#define DEBUG_TYPE "mlir-tblgen-rewritergen"
46
47namespace llvm {
48template <>
49struct format_provider<mlir::tblgen::Pattern::IdentifierLine> {
50 static void format(const mlir::tblgen::Pattern::IdentifierLine &v,
51 raw_ostream &os, StringRef style) {
52 os << v.first << ":" << v.second;
53 }
54};
55} // namespace llvm
56
57//===----------------------------------------------------------------------===//
58// PatternEmitter
59//===----------------------------------------------------------------------===//
60
61namespace {
62
63class StaticMatcherHelper;
64
65class PatternEmitter {
66public:
67 PatternEmitter(const Record *pat, RecordOperatorMap *mapper, raw_ostream &os,
68 StaticMatcherHelper &helper);
69
70 // Emits the mlir::RewritePattern struct named `rewriteName`.
71 void emit(StringRef rewriteName);
72
73 // Emits the static function of DAG matcher.
74 void emitStaticMatcher(DagNode tree, std::string funcName);
75
76private:
77 // Emits the code for matching ops.
78 void emitMatchLogic(DagNode tree, StringRef opName);
79
80 // Emits the code for rewriting ops.
81 void emitRewriteLogic();
82
83 //===--------------------------------------------------------------------===//
84 // Match utilities
85 //===--------------------------------------------------------------------===//
86
87 // Emits C++ statements for matching the DAG structure.
88 void emitMatch(DagNode tree, StringRef name, int depth);
89
90 // Emit C++ function call to static DAG matcher.
91 void emitStaticMatchCall(DagNode tree, StringRef name);
92
93 // Emit C++ function call to static type/attribute constraint function.
94 void emitStaticVerifierCall(StringRef funcName, StringRef opName,
95 StringRef arg, StringRef failureStr);
96
97 // Emits C++ statements for matching using a native code call.
98 void emitNativeCodeMatch(DagNode tree, StringRef name, int depth);
99
100 // Emits C++ statements for matching the op constrained by the given DAG
101 // `tree` returning the op's variable name.
102 void emitOpMatch(DagNode tree, StringRef opName, int depth);
103
104 // Emits C++ statements for matching the `argIndex`-th argument of the given
105 // DAG `tree` as an operand. `operandName` and `operandMatcher` indicate the
106 // bound name and the constraint of the operand respectively.
107 void emitOperandMatch(DagNode tree, StringRef opName, StringRef operandName,
108 int operandIndex, DagLeaf operandMatcher,
109 StringRef argName, int argIndex,
110 std::optional<int> variadicSubIndex);
111
112 // Emits C++ statements for matching the operands which can be matched in
113 // either order.
114 void emitEitherOperandMatch(DagNode tree, DagNode eitherArgTree,
115 StringRef opName, int argIndex, int &operandIndex,
116 int depth);
117
118 // Emits C++ statements for matching a variadic operand.
119 void emitVariadicOperandMatch(DagNode tree, DagNode variadicArgTree,
120 StringRef opName, int argIndex,
121 int &operandIndex, int depth);
122
123 // Emits C++ statements for matching the `argIndex`-th argument of the given
124 // DAG `tree` as an attribute.
125 void emitAttributeMatch(DagNode tree, StringRef castedName, int argIndex,
126 int depth);
127
128 // Emits C++ statements for matching the `argIndex`-th argument of the given
129 // DAG `tree` as a property.
130 void emitPropertyMatch(DagNode tree, StringRef castedName, int argIndex,
131 int depth);
132
133 // Emits C++ for checking a match with a corresponding match failure
134 // diagnostic.
135 void emitMatchCheck(StringRef opName, const FmtObjectBase &matchFmt,
136 const llvm::formatv_object_base &failureFmt);
137
138 // Emits C++ for checking a match with a corresponding match failure
139 // diagnostics.
140 void emitMatchCheck(StringRef opName, const std::string &matchStr,
141 const std::string &failureStr);
142
143 //===--------------------------------------------------------------------===//
144 // Rewrite utilities
145 //===--------------------------------------------------------------------===//
146
147 // The entry point for handling a result pattern rooted at `resultTree`. This
148 // method dispatches to concrete handlers according to `resultTree`'s kind and
149 // returns a symbol representing the whole value pack. Callers are expected to
150 // further resolve the symbol according to the specific use case.
151 //
152 // `depth` is the nesting level of `resultTree`; 0 means top-level result
153 // pattern. For top-level result pattern, `resultIndex` indicates which result
154 // of the matched root op this pattern is intended to replace, which can be
155 // used to deduce the result type of the op generated from this result
156 // pattern.
157 std::string handleResultPattern(DagNode resultTree, int resultIndex,
158 int depth);
159
160 // Emits the C++ statement to replace the matched DAG with a value built via
161 // calling native C++ code.
162 std::string handleReplaceWithNativeCodeCall(DagNode resultTree, int depth);
163
164 // Returns the symbol of the old value serving as the replacement.
165 StringRef handleReplaceWithValue(DagNode tree);
166
167 // Emits the C++ statement to replace the matched DAG with an array of
168 // matched values.
169 std::string handleVariadic(DagNode tree, int depth);
170
171 // Trailing directives are used at the end of DAG node argument lists to
172 // specify additional behaviour for op matchers and creators, etc.
173 struct TrailingDirectives {
174 // DAG node containing the `location` directive. Null if there is none.
175 DagNode location;
176
177 // DAG node containing the `returnType` directive. Null if there is none.
178 DagNode returnType;
179
180 // Number of found trailing directives.
181 int numDirectives;
182 };
183
184 // Collect any trailing directives.
185 TrailingDirectives getTrailingDirectives(DagNode tree);
186
187 // Returns the location value to use.
188 std::string getLocation(TrailingDirectives &tail);
189
190 // Returns the location value to use.
191 std::string handleLocationDirective(DagNode tree);
192
193 // Emit return type argument.
194 std::string handleReturnTypeArg(DagNode returnType, int i, int depth);
195
196 // Emits the C++ statement to build a new op out of the given DAG `tree` and
197 // returns the variable name that this op is assigned to. If the root op in
198 // DAG `tree` has a specified name, the created op will be assigned to a
199 // variable of the given name. Otherwise, a unique name will be used as the
200 // result value name.
201 std::string handleOpCreation(DagNode tree, int resultIndex, int depth);
202
203 using ChildNodeIndexNameMap = DenseMap<unsigned, std::string>;
204
205 // Emits a local variable for each value and attribute to be used for creating
206 // an op.
207 void createSeparateLocalVarsForOpArgs(DagNode node,
208 ChildNodeIndexNameMap &childNodeNames);
209
210 // Emits the concrete arguments used to call an op's builder.
211 void supplyValuesForOpArgs(DagNode node,
212 const ChildNodeIndexNameMap &childNodeNames,
213 int depth);
214
215 // Emits the local variables for holding all values as a whole and all named
216 // attributes as a whole to be used for creating an op.
217 void createAggregateLocalVarsForOpArgs(
218 DagNode node, const ChildNodeIndexNameMap &childNodeNames, int depth);
219
220 // Returns the C++ expression to construct a constant attribute of the given
221 // `value` for the given attribute kind `attr`.
222 std::string handleConstantAttr(Attribute attr, const Twine &value);
223
224 // Returns the C++ expression to build an argument from the given DAG `leaf`.
225 // `patArgName` is used to bound the argument to the source pattern.
226 std::string handleOpArgument(DagLeaf leaf, StringRef patArgName);
227
228 //===--------------------------------------------------------------------===//
229 // General utilities
230 //===--------------------------------------------------------------------===//
231
232 // Collects all of the operations within the given dag tree.
233 void collectOps(DagNode tree, llvm::SmallPtrSetImpl<const Operator *> &ops);
234
235 // Returns a unique symbol for a local variable of the given `op`.
236 std::string getUniqueSymbol(const Operator *op);
237
238 //===--------------------------------------------------------------------===//
239 // Symbol utilities
240 //===--------------------------------------------------------------------===//
241
242 // Returns how many static values the given DAG `node` correspond to.
243 int getNodeValueCount(DagNode node);
244
245private:
246 // Pattern instantiation location followed by the location of multiclass
247 // prototypes used. This is intended to be used as a whole to
248 // PrintFatalError() on errors.
249 ArrayRef<SMLoc> loc;
250
251 // Op's TableGen Record to wrapper object.
252 RecordOperatorMap *opMap;
253
254 // Handy wrapper for pattern being emitted.
255 Pattern pattern;
256
257 // Map for all bound symbols' info.
258 SymbolInfoMap symbolInfoMap;
259
260 StaticMatcherHelper &staticMatcherHelper;
261
262 // The next unused ID for newly created values.
263 unsigned nextValueId = 0;
264
265 raw_indented_ostream os;
266
267 // Format contexts containing placeholder substitutions.
268 FmtContext fmtCtx;
269};
270
271// Tracks DagNode's reference multiple times across patterns. Enables generating
272// static matcher functions for DagNode's referenced multiple times rather than
273// inlining them.
274class StaticMatcherHelper {
275public:
276 StaticMatcherHelper(raw_ostream &os, const RecordKeeper &records,
277 RecordOperatorMap &mapper);
278
279 // Determine if we should inline the match logic or delegate to a static
280 // function.
281 bool useStaticMatcher(DagNode node) {
282 // either/variadic node must be associated to the parentOp, thus we can't
283 // emit a static matcher rooted at them.
284 if (node.isEither() || node.isVariadic())
285 return false;
286
287 return refStats[node] > kStaticMatcherThreshold;
288 }
289
290 // Get the name of the static DAG matcher function corresponding to the node.
291 std::string getMatcherName(DagNode node) {
292 assert(useStaticMatcher(node));
293 return matcherNames[node];
294 }
295
296 // Get the name of static type/attribute verification function.
297 StringRef getVerifierName(DagLeaf leaf);
298
299 // Collect the `Record`s, i.e., the DRR, so that we can get the information of
300 // the duplicated DAGs.
301 void addPattern(const Record *record);
302
303 // Emit all static functions of DAG Matcher.
304 void populateStaticMatchers(raw_ostream &os);
305
306 // Emit all static functions for Constraints.
307 void populateStaticConstraintFunctions(raw_ostream &os);
308
309private:
310 static constexpr unsigned kStaticMatcherThreshold = 1;
311
312 // Consider two patterns as down below,
313 // DagNode_Root_A DagNode_Root_B
314 // \ \
315 // DagNode_C DagNode_C
316 // \ \
317 // DagNode_D DagNode_D
318 //
319 // DagNode_Root_A and DagNode_Root_B share the same subtree which consists of
320 // DagNode_C and DagNode_D. Both DagNode_C and DagNode_D are referenced
321 // multiple times so we'll have static matchers for both of them. When we're
322 // emitting the match logic for DagNode_C, we will check if DagNode_D has the
323 // static matcher generated. If so, then we'll generate a call to the
324 // function, inline otherwise. In this case, inlining is not what we want. As
325 // a result, generate the static matcher in topological order to ensure all
326 // the dependent static matchers are generated and we can avoid accidentally
327 // inlining.
328 //
329 // The topological order of all the DagNodes among all patterns.
330 SmallVector<std::pair<DagNode, const Record *>> topologicalOrder;
331
332 RecordOperatorMap &opMap;
333
334 // Records of the static function name of each DagNode
335 DenseMap<DagNode, std::string> matcherNames;
336
337 // After collecting all the DagNode in each pattern, `refStats` records the
338 // number of users for each DagNode. We will generate the static matcher for a
339 // DagNode while the number of users exceeds a certain threshold.
340 DenseMap<DagNode, unsigned> refStats;
341
342 // Number of static matcher generated. This is used to generate a unique name
343 // for each DagNode.
344 int staticMatcherCounter = 0;
345
346 // The DagLeaf which contains type, attr, or prop constraint.
347 SetVector<DagLeaf> constraints;
348
349 // Static type/attribute verification function emitter.
350 StaticVerifierFunctionEmitter staticVerifierEmitter;
351};
352
353} // namespace
354
355PatternEmitter::PatternEmitter(const Record *pat, RecordOperatorMap *mapper,
356 raw_ostream &os, StaticMatcherHelper &helper)
357 : loc(pat->getLoc()), opMap(mapper), pattern(pat, mapper),
358 symbolInfoMap(pat->getLoc()), staticMatcherHelper(helper), os(os) {
359 fmtCtx.withBuilder(subst: "rewriter");
360}
361
362std::string PatternEmitter::handleConstantAttr(Attribute attr,
363 const Twine &value) {
364 if (!attr.isConstBuildable())
365 PrintFatalError(ErrorLoc: loc, Msg: "Attribute " + attr.getAttrDefName() +
366 " does not have the 'constBuilderCall' field");
367
368 // TODO: Verify the constants here
369 return std::string(tgfmt(fmt: attr.getConstBuilderTemplate(), ctx: &fmtCtx, vals: value));
370}
371
372void PatternEmitter::emitStaticMatcher(DagNode tree, std::string funcName) {
373 os << formatv(
374 Fmt: "static ::llvm::LogicalResult {0}(::mlir::PatternRewriter &rewriter, "
375 "::mlir::Operation *op0, ::llvm::SmallVector<::mlir::Operation "
376 "*, 4> &tblgen_ops",
377 Vals&: funcName);
378
379 // We pass the reference of the variables that need to be captured. Hence we
380 // need to collect all the symbols in the tree first.
381 pattern.collectBoundSymbols(tree, infoMap&: symbolInfoMap, /*isSrcPattern=*/true);
382 symbolInfoMap.assignUniqueAlternativeNames();
383 for (const auto &info : symbolInfoMap)
384 os << formatv(Fmt: ", {0}", Vals: info.second.getArgDecl(name: info.first));
385
386 os << ") {\n";
387 os.indent();
388 os << "(void)tblgen_ops;\n";
389
390 // Note that a static matcher is considered at least one step from the match
391 // entry.
392 emitMatch(tree, name: "op0", /*depth=*/1);
393
394 os << "return ::mlir::success();\n";
395 os.unindent();
396 os << "}\n\n";
397}
398
399// Helper function to match patterns.
400void PatternEmitter::emitMatch(DagNode tree, StringRef name, int depth) {
401 if (tree.isNativeCodeCall()) {
402 emitNativeCodeMatch(tree, name, depth);
403 return;
404 }
405
406 if (tree.isOperation()) {
407 emitOpMatch(tree, opName: name, depth);
408 return;
409 }
410
411 PrintFatalError(ErrorLoc: loc, Msg: "encountered non-op, non-NativeCodeCall match.");
412}
413
414void PatternEmitter::emitStaticMatchCall(DagNode tree, StringRef opName) {
415 std::string funcName = staticMatcherHelper.getMatcherName(node: tree);
416 os << formatv(Fmt: "if(::mlir::failed({0}(rewriter, {1}, tblgen_ops", Vals&: funcName,
417 Vals&: opName);
418
419 // TODO(chiahungduan): Add a lookupBoundSymbols() to do the subtree lookup in
420 // one pass.
421
422 // In general, bound symbol should have the unique name in the pattern but
423 // for the operand, binding same symbol to multiple operands imply a
424 // constraint at the same time. In this case, we will rename those operands
425 // with different names. As a result, we need to collect all the symbolInfos
426 // from the DagNode then get the updated name of the local variables from the
427 // global symbolInfoMap.
428
429 // Collect all the bound symbols in the Dag
430 SymbolInfoMap localSymbolMap(loc);
431 pattern.collectBoundSymbols(tree, infoMap&: localSymbolMap, /*isSrcPattern=*/true);
432
433 for (const auto &[name, symbolInfo] : localSymbolMap) {
434 auto ret = symbolInfoMap.findBoundSymbol(key: name, symbolInfo);
435 os << formatv(Fmt: ", {0}", Vals: ret->second.getVarName(name));
436 }
437
438 os << "))) {\n";
439 os.scope().os << "return ::mlir::failure();\n";
440 os << "}\n";
441}
442
443void PatternEmitter::emitStaticVerifierCall(StringRef funcName,
444 StringRef opName, StringRef arg,
445 StringRef failureStr) {
446 os << formatv(Fmt: "if(::mlir::failed({0}(rewriter, {1}, {2}, {3}))) {{\n",
447 Vals&: funcName, Vals&: opName, Vals&: arg, Vals&: failureStr);
448 os.scope().os << "return ::mlir::failure();\n";
449 os << "}\n";
450}
451
452// Helper function to match patterns.
453void PatternEmitter::emitNativeCodeMatch(DagNode tree, StringRef opName,
454 int depth) {
455 LLVM_DEBUG(llvm::dbgs() << "handle NativeCodeCall matcher pattern: ");
456 LLVM_DEBUG(tree.print(llvm::dbgs()));
457 LLVM_DEBUG(llvm::dbgs() << '\n');
458
459 // The order of generating static matcher follows the topological order so
460 // that for every dependent DagNode already have their static matcher
461 // generated if needed. The reason we check if `getMatcherName(tree).empty()`
462 // is when we are generating the static matcher for a DagNode itself. In this
463 // case, we need to emit the function body rather than a function call.
464 if (staticMatcherHelper.useStaticMatcher(node: tree) &&
465 !staticMatcherHelper.getMatcherName(node: tree).empty()) {
466 emitStaticMatchCall(tree, opName);
467
468 // NativeCodeCall will never be at depth 0 so that we don't need to catch
469 // the root operation as emitOpMatch();
470
471 return;
472 }
473
474 // TODO(suderman): iterate through arguments, determine their types, output
475 // names.
476 SmallVector<std::string, 8> capture;
477
478 raw_indented_ostream::DelimitedScope scope(os);
479
480 for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
481 std::string argName = formatv(Fmt: "arg{0}_{1}", Vals&: depth, Vals&: i);
482 if (DagNode argTree = tree.getArgAsNestedDag(index: i)) {
483 if (argTree.isEither())
484 PrintFatalError(ErrorLoc: loc, Msg: "NativeCodeCall cannot have `either` operands");
485 if (argTree.isVariadic())
486 PrintFatalError(ErrorLoc: loc, Msg: "NativeCodeCall cannot have `variadic` operands");
487
488 os << "::mlir::Value " << argName << ";\n";
489 } else {
490 auto leaf = tree.getArgAsLeaf(index: i);
491 if (leaf.isAttrMatcher() || leaf.isConstantAttr()) {
492 os << "::mlir::Attribute " << argName << ";\n";
493 } else if (leaf.isPropMatcher()) {
494 StringRef interfaceType = leaf.getAsPropConstraint().getInterfaceType();
495 if (interfaceType.empty())
496 PrintFatalError(ErrorLoc: loc, Msg: "NativeCodeCall cannot have a property operand "
497 "with unspecified interface type");
498 os << interfaceType << " " << argName;
499 if (leaf.isPropDefinition()) {
500 Property propDef = leaf.getAsProperty();
501 // Ensure properties that aren't zero-arg-constructable still work.
502 if (propDef.hasDefaultValue())
503 os << " = " << propDef.getDefaultValue();
504 }
505 os << ";\n";
506 } else {
507 os << "::mlir::Value " << argName << ";\n";
508 }
509 }
510
511 capture.push_back(Elt: std::move(argName));
512 }
513
514 auto tail = getTrailingDirectives(tree);
515 if (tail.returnType)
516 PrintFatalError(ErrorLoc: loc, Msg: "`NativeCodeCall` cannot have return type specifier");
517 auto locToUse = getLocation(tail);
518
519 auto fmt = tree.getNativeCodeTemplate();
520 if (fmt.count(Str: "$_self") != 1)
521 PrintFatalError(ErrorLoc: loc, Msg: "NativeCodeCall must have $_self as argument for "
522 "passing the defining Operation");
523
524 auto nativeCodeCall = std::string(
525 tgfmtv(fmt, ctx: &fmtCtx.addSubst(placeholder: "_loc", subst: locToUse).withSelf(subst: opName.str()),
526 params: static_cast<ArrayRef<std::string>>(capture)));
527
528 emitMatchCheck(opName, matchStr: formatv(Fmt: "!::mlir::failed({0})", Vals&: nativeCodeCall),
529 failureStr: formatv(Fmt: "\"{0} return ::mlir::failure\"", Vals&: nativeCodeCall));
530
531 for (int i = 0, e = tree.getNumArgs() - tail.numDirectives; i != e; ++i) {
532 auto name = tree.getArgName(index: i);
533 if (!name.empty() && name != "_") {
534 os << formatv(Fmt: "{0} = {1};\n", Vals&: name, Vals&: capture[i]);
535 }
536 }
537
538 for (int i = 0, e = tree.getNumArgs() - tail.numDirectives; i != e; ++i) {
539 std::string argName = capture[i];
540
541 // Handle nested DAG construct first
542 if (tree.getArgAsNestedDag(index: i)) {
543 PrintFatalError(
544 ErrorLoc: loc, Msg: formatv(Fmt: "Matching nested tree in NativeCodecall not support for "
545 "{0} as arg {1}",
546 Vals&: argName, Vals&: i));
547 }
548
549 DagLeaf leaf = tree.getArgAsLeaf(index: i);
550
551 // The parameter for native function doesn't bind any constraints.
552 if (leaf.isUnspecified())
553 continue;
554
555 auto constraint = leaf.getAsConstraint();
556
557 std::string self;
558 if (leaf.isAttrMatcher() || leaf.isConstantAttr() || leaf.isPropMatcher())
559 self = argName;
560 else
561 self = formatv(Fmt: "{0}.getType()", Vals&: argName);
562 StringRef verifier = staticMatcherHelper.getVerifierName(leaf);
563 emitStaticVerifierCall(
564 funcName: verifier, opName, arg: self,
565 failureStr: formatv(Fmt: "\"operand {0} of native code call '{1}' failed to satisfy "
566 "constraint: "
567 "'{2}'\"",
568 Vals&: i, Vals: tree.getNativeCodeTemplate(),
569 Vals: escapeString(value: constraint.getSummary()))
570 .str());
571 }
572
573 LLVM_DEBUG(llvm::dbgs() << "done emitting match for native code call\n");
574}
575
576// Helper function to match patterns.
577void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
578 Operator &op = tree.getDialectOp(mapper: opMap);
579 LLVM_DEBUG(llvm::dbgs() << "start emitting match for op '"
580 << op.getOperationName() << "' at depth " << depth
581 << '\n');
582
583 auto getCastedName = [depth]() -> std::string {
584 return formatv(Fmt: "castedOp{0}", Vals: depth);
585 };
586
587 // The order of generating static matcher follows the topological order so
588 // that for every dependent DagNode already have their static matcher
589 // generated if needed. The reason we check if `getMatcherName(tree).empty()`
590 // is when we are generating the static matcher for a DagNode itself. In this
591 // case, we need to emit the function body rather than a function call.
592 if (staticMatcherHelper.useStaticMatcher(node: tree) &&
593 !staticMatcherHelper.getMatcherName(node: tree).empty()) {
594 emitStaticMatchCall(tree, opName);
595 // In the codegen of rewriter, we suppose that castedOp0 will capture the
596 // root operation. Manually add it if the root DagNode is a static matcher.
597 if (depth == 0)
598 os << formatv(Fmt: "auto {2} = ::llvm::dyn_cast_or_null<{1}>({0}); "
599 "(void){2};\n",
600 Vals&: opName, Vals: op.getQualCppClassName(), Vals: getCastedName());
601 return;
602 }
603
604 std::string castedName = getCastedName();
605 os << formatv(Fmt: "auto {0} = ::llvm::dyn_cast<{2}>({1}); "
606 "(void){0};\n",
607 Vals&: castedName, Vals&: opName, Vals: op.getQualCppClassName());
608
609 // Skip the operand matching at depth 0 as the pattern rewriter already does.
610 if (depth != 0)
611 emitMatchCheck(opName, /*matchStr=*/castedName,
612 failureStr: formatv(Fmt: "\"{0} is not {1} type\"", Vals&: castedName,
613 Vals: op.getQualCppClassName()));
614
615 // If the operand's name is set, set to that variable.
616 auto name = tree.getSymbol();
617 if (!name.empty())
618 os << formatv(Fmt: "{0} = {1};\n", Vals&: name, Vals&: castedName);
619
620 for (int i = 0, opArgIdx = 0, e = tree.getNumArgs(), nextOperand = 0; i != e;
621 ++i, ++opArgIdx) {
622 auto opArg = op.getArg(index: opArgIdx);
623 std::string argName = formatv(Fmt: "op{0}", Vals: depth + 1);
624
625 // Handle nested DAG construct first
626 if (DagNode argTree = tree.getArgAsNestedDag(index: i)) {
627 if (argTree.isEither()) {
628 emitEitherOperandMatch(tree, eitherArgTree: argTree, opName: castedName, argIndex: opArgIdx, operandIndex&: nextOperand,
629 depth);
630 ++opArgIdx;
631 continue;
632 }
633 if (auto *operand =
634 llvm::dyn_cast_if_present<NamedTypeConstraint *>(Val&: opArg)) {
635 if (argTree.isVariadic()) {
636 if (!operand->isVariadic()) {
637 auto error = formatv(Fmt: "variadic DAG construct can't match op {0}'s "
638 "non-variadic operand #{1}",
639 Vals: op.getOperationName(), Vals&: opArgIdx);
640 PrintFatalError(ErrorLoc: loc, Msg: error);
641 }
642 emitVariadicOperandMatch(tree, variadicArgTree: argTree, opName: castedName, argIndex: opArgIdx,
643 operandIndex&: nextOperand, depth);
644 ++nextOperand;
645 continue;
646 }
647 if (operand->isVariableLength()) {
648 auto error = formatv(Fmt: "use nested DAG construct to match op {0}'s "
649 "variadic operand #{1} unsupported now",
650 Vals: op.getOperationName(), Vals&: opArgIdx);
651 PrintFatalError(ErrorLoc: loc, Msg: error);
652 }
653 }
654
655 os << "{\n";
656
657 // Attributes don't count for getODSOperands.
658 // TODO: Operand is a Value, check if we should remove `getDefiningOp()`.
659 os.indent() << formatv(
660 Fmt: "auto *{0} = "
661 "(*{1}.getODSOperands({2}).begin()).getDefiningOp();\n",
662 Vals&: argName, Vals&: castedName, Vals&: nextOperand);
663 // Null check of operand's definingOp
664 emitMatchCheck(
665 opName: castedName, /*matchStr=*/argName,
666 failureStr: formatv(Fmt: "\"There's no operation that defines operand {0} of {1}\"",
667 Vals: nextOperand++, Vals&: castedName));
668 emitMatch(tree: argTree, name: argName, depth: depth + 1);
669 os << formatv(Fmt: "tblgen_ops.push_back({0});\n", Vals&: argName);
670 os.unindent() << "}\n";
671 continue;
672 }
673
674 // Next handle DAG leaf: operand or attribute
675 if (isa<NamedTypeConstraint *>(Val: opArg)) {
676 auto operandName =
677 formatv(Fmt: "{0}.getODSOperands({1})", Vals&: castedName, Vals&: nextOperand);
678 emitOperandMatch(tree, opName: castedName, operandName: operandName.str(), operandIndex: nextOperand,
679 /*operandMatcher=*/tree.getArgAsLeaf(index: i),
680 /*argName=*/tree.getArgName(index: i), argIndex: opArgIdx,
681 /*variadicSubIndex=*/std::nullopt);
682 ++nextOperand;
683 } else if (isa<NamedAttribute *>(Val: opArg)) {
684 emitAttributeMatch(tree, castedName, argIndex: opArgIdx, depth);
685 } else if (isa<NamedProperty *>(Val: opArg)) {
686 emitPropertyMatch(tree, castedName, argIndex: opArgIdx, depth);
687 } else {
688 PrintFatalError(ErrorLoc: loc, Msg: "unhandled case when matching op");
689 }
690 }
691 LLVM_DEBUG(llvm::dbgs() << "done emitting match for op '"
692 << op.getOperationName() << "' at depth " << depth
693 << '\n');
694}
695
696void PatternEmitter::emitOperandMatch(DagNode tree, StringRef opName,
697 StringRef operandName, int operandIndex,
698 DagLeaf operandMatcher, StringRef argName,
699 int argIndex,
700 std::optional<int> variadicSubIndex) {
701 Operator &op = tree.getDialectOp(mapper: opMap);
702 NamedTypeConstraint operand = op.getOperand(index: operandIndex);
703
704 // If a constraint is specified, we need to generate C++ statements to
705 // check the constraint.
706 if (!operandMatcher.isUnspecified()) {
707 if (!operandMatcher.isOperandMatcher())
708 PrintFatalError(
709 ErrorLoc: loc, Msg: formatv(Fmt: "the {1}-th argument of op '{0}' should be an operand",
710 Vals: op.getOperationName(), Vals: argIndex + 1));
711
712 // Only need to verify if the matcher's type is different from the one
713 // of op definition.
714 Constraint constraint = operandMatcher.getAsConstraint();
715 if (operand.constraint != constraint) {
716 if (operand.isVariableLength()) {
717 auto error = formatv(
718 Fmt: "further constrain op {0}'s variadic operand #{1} unsupported now",
719 Vals: op.getOperationName(), Vals&: argIndex);
720 PrintFatalError(ErrorLoc: loc, Msg: error);
721 }
722 auto self = formatv(Fmt: "(*{0}.begin()).getType()", Vals&: operandName);
723 StringRef verifier = staticMatcherHelper.getVerifierName(leaf: operandMatcher);
724 emitStaticVerifierCall(
725 funcName: verifier, opName, arg: self.str(),
726 failureStr: formatv(
727 Fmt: "\"operand {0} of op '{1}' failed to satisfy constraint: '{2}'\"",
728 Vals&: operandIndex, Vals: op.getOperationName(),
729 Vals: escapeString(value: constraint.getSummary()))
730 .str());
731 }
732 }
733
734 // Capture the value
735 // `$_` is a special symbol to ignore op argument matching.
736 if (!argName.empty() && argName != "_") {
737 auto res = symbolInfoMap.findBoundSymbol(key: argName, node: tree, op, argIndex,
738 variadicSubIndex);
739 if (res == symbolInfoMap.end())
740 PrintFatalError(ErrorLoc: loc, Msg: formatv(Fmt: "symbol not found: {0}", Vals&: argName));
741
742 os << formatv(Fmt: "{0} = {1};\n", Vals: res->second.getVarName(name: argName), Vals&: operandName);
743 }
744}
745
746void PatternEmitter::emitEitherOperandMatch(DagNode tree, DagNode eitherArgTree,
747 StringRef opName, int argIndex,
748 int &operandIndex, int depth) {
749 constexpr int numEitherArgs = 2;
750 if (eitherArgTree.getNumArgs() != numEitherArgs)
751 PrintFatalError(ErrorLoc: loc, Msg: "`either` only supports grouping two operands");
752
753 Operator &op = tree.getDialectOp(mapper: opMap);
754
755 std::string codeBuffer;
756 llvm::raw_string_ostream tblgenOps(codeBuffer);
757
758 std::string lambda = formatv(Fmt: "eitherLambda{0}", Vals&: depth);
759 os << formatv(
760 Fmt: "auto {0} = [&](::mlir::OperandRange v0, ::mlir::OperandRange v1) {{\n",
761 Vals&: lambda);
762
763 os.indent();
764
765 for (int i = 0; i < numEitherArgs; ++i, ++argIndex) {
766 if (DagNode argTree = eitherArgTree.getArgAsNestedDag(index: i)) {
767 if (argTree.isEither())
768 PrintFatalError(ErrorLoc: loc, Msg: "either cannot be nested");
769
770 std::string argName = formatv(Fmt: "local_op_{0}", Vals&: i).str();
771
772 os << formatv(Fmt: "auto {0} = (*v{1}.begin()).getDefiningOp();\n", Vals&: argName,
773 Vals&: i);
774
775 // Indent emitMatchCheck and emitMatch because they declare local
776 // variables.
777 os << "{\n";
778 os.indent();
779
780 emitMatchCheck(
781 opName, /*matchStr=*/argName,
782 failureStr: formatv(Fmt: "\"There's no operation that defines operand {0} of {1}\"",
783 Vals: operandIndex++, Vals&: opName));
784 emitMatch(tree: argTree, name: argName, depth: depth + 1);
785
786 os.unindent() << "}\n";
787
788 // `tblgen_ops` is used to collect the matched operations. In either, we
789 // need to queue the operation only if the matching success. Thus we emit
790 // the code at the end.
791 tblgenOps << formatv(Fmt: "tblgen_ops.push_back({0});\n", Vals&: argName);
792 } else if (isa<NamedTypeConstraint *>(Val: op.getArg(index: argIndex))) {
793 emitOperandMatch(tree, opName, /*operandName=*/formatv(Fmt: "v{0}", Vals&: i).str(),
794 operandIndex,
795 /*operandMatcher=*/eitherArgTree.getArgAsLeaf(index: i),
796 /*argName=*/eitherArgTree.getArgName(index: i), argIndex,
797 /*variadicSubIndex=*/std::nullopt);
798 ++operandIndex;
799 } else {
800 PrintFatalError(ErrorLoc: loc, Msg: "either can only be applied on operand");
801 }
802 }
803
804 os << tblgenOps.str();
805 os << "return ::mlir::success();\n";
806 os.unindent() << "};\n";
807
808 os << "{\n";
809 os.indent();
810
811 os << formatv(Fmt: "auto eitherOperand0 = {0}.getODSOperands({1});\n", Vals&: opName,
812 Vals: operandIndex - 2);
813 os << formatv(Fmt: "auto eitherOperand1 = {0}.getODSOperands({1});\n", Vals&: opName,
814 Vals: operandIndex - 1);
815
816 os << formatv(Fmt: "if(::mlir::failed({0}(eitherOperand0, eitherOperand1)) && "
817 "::mlir::failed({0}(eitherOperand1, "
818 "eitherOperand0)))\n",
819 Vals&: lambda);
820 os.indent() << "return ::mlir::failure();\n";
821
822 os.unindent().unindent() << "}\n";
823}
824
825void PatternEmitter::emitVariadicOperandMatch(DagNode tree,
826 DagNode variadicArgTree,
827 StringRef opName, int argIndex,
828 int &operandIndex, int depth) {
829 Operator &op = tree.getDialectOp(mapper: opMap);
830
831 os << "{\n";
832 os.indent();
833
834 os << formatv(Fmt: "auto variadic_operand_range = {0}.getODSOperands({1});\n",
835 Vals&: opName, Vals&: operandIndex);
836 os << formatv(Fmt: "if (variadic_operand_range.size() != {0}) "
837 "return ::mlir::failure();\n",
838 Vals: variadicArgTree.getNumArgs());
839
840 StringRef variadicTreeName = variadicArgTree.getSymbol();
841 if (!variadicTreeName.empty()) {
842 auto res =
843 symbolInfoMap.findBoundSymbol(key: variadicTreeName, node: tree, op, argIndex,
844 /*variadicSubIndex=*/std::nullopt);
845 if (res == symbolInfoMap.end())
846 PrintFatalError(ErrorLoc: loc, Msg: formatv(Fmt: "symbol not found: {0}", Vals&: variadicTreeName));
847
848 os << formatv(Fmt: "{0} = variadic_operand_range;\n",
849 Vals: res->second.getVarName(name: variadicTreeName));
850 }
851
852 for (int i = 0; i < variadicArgTree.getNumArgs(); ++i) {
853 if (DagNode argTree = variadicArgTree.getArgAsNestedDag(index: i)) {
854 if (!argTree.isOperation())
855 PrintFatalError(ErrorLoc: loc, Msg: "variadic only accepts operation sub-dags");
856
857 os << "{\n";
858 os.indent();
859
860 std::string argName = formatv(Fmt: "local_op_{0}", Vals&: i).str();
861 os << formatv(Fmt: "auto *{0} = "
862 "variadic_operand_range[{1}].getDefiningOp();\n",
863 Vals&: argName, Vals&: i);
864 emitMatchCheck(
865 opName, /*matchStr=*/argName,
866 failureStr: formatv(Fmt: "\"There's no operation that defines variadic operand "
867 "{0} (variadic sub-opearnd #{1}) of {2}\"",
868 Vals&: operandIndex, Vals&: i, Vals&: opName));
869 emitMatch(tree: argTree, name: argName, depth: depth + 1);
870 os << formatv(Fmt: "tblgen_ops.push_back({0});\n", Vals&: argName);
871
872 os.unindent() << "}\n";
873 } else if (isa<NamedTypeConstraint *>(Val: op.getArg(index: argIndex))) {
874 auto operandName = formatv(Fmt: "variadic_operand_range.slice({0}, 1)", Vals&: i);
875 emitOperandMatch(tree, opName, operandName: operandName.str(), operandIndex,
876 /*operandMatcher=*/variadicArgTree.getArgAsLeaf(index: i),
877 /*argName=*/variadicArgTree.getArgName(index: i), argIndex, variadicSubIndex: i);
878 } else {
879 PrintFatalError(ErrorLoc: loc, Msg: "variadic can only be applied on operand");
880 }
881 }
882
883 os.unindent() << "}\n";
884}
885
886void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef castedName,
887 int argIndex, int depth) {
888 Operator &op = tree.getDialectOp(mapper: opMap);
889 auto *namedAttr = cast<NamedAttribute *>(Val: op.getArg(index: argIndex));
890 const auto &attr = namedAttr->attr;
891
892 os << "{\n";
893 os.indent() << formatv(
894 Fmt: "[[maybe_unused]] auto tblgen_attr = {0}.getProperties().{1}();\n",
895 Vals&: castedName, Vals: op.getGetterName(name: namedAttr->name));
896
897 // TODO: This should use getter method to avoid duplication.
898 if (attr.hasDefaultValue()) {
899 os << "if (!tblgen_attr) tblgen_attr = "
900 << std::string(tgfmt(fmt: attr.getConstBuilderTemplate(), ctx: &fmtCtx,
901 vals: tgfmt(fmt: attr.getDefaultValue(), ctx: &fmtCtx)))
902 << ";\n";
903 } else if (attr.isOptional()) {
904 // For a missing attribute that is optional according to definition, we
905 // should just capture a mlir::Attribute() to signal the missing state.
906 // That is precisely what getDiscardableAttr() returns on missing
907 // attributes.
908 } else {
909 emitMatchCheck(opName: castedName, matchFmt: tgfmt(fmt: "tblgen_attr", ctx: &fmtCtx),
910 failureFmt: formatv(Fmt: "\"expected op '{0}' to have attribute '{1}' "
911 "of type '{2}'\"",
912 Vals: op.getOperationName(), Vals&: namedAttr->name,
913 Vals: attr.getStorageType()));
914 }
915
916 auto matcher = tree.getArgAsLeaf(index: argIndex);
917 if (!matcher.isUnspecified()) {
918 if (!matcher.isAttrMatcher()) {
919 PrintFatalError(
920 ErrorLoc: loc, Msg: formatv(Fmt: "the {1}-th argument of op '{0}' should be an attribute",
921 Vals: op.getOperationName(), Vals: argIndex + 1));
922 }
923
924 // If a constraint is specified, we need to generate function call to its
925 // static verifier.
926 StringRef verifier = staticMatcherHelper.getVerifierName(leaf: matcher);
927 if (attr.isOptional()) {
928 // Avoid dereferencing null attribute. This is using a simple heuristic to
929 // avoid common cases of attempting to dereference null attribute. This
930 // will return where there is no check if attribute is null unless the
931 // attribute's value is not used.
932 // FIXME: This could be improved as some null dereferences could slip
933 // through.
934 if (!StringRef(matcher.getConditionTemplate()).contains(Other: "!$_self") &&
935 StringRef(matcher.getConditionTemplate()).contains(Other: "$_self")) {
936 os << "if (!tblgen_attr) return ::mlir::failure();\n";
937 }
938 }
939 emitStaticVerifierCall(
940 funcName: verifier, opName: castedName, arg: "tblgen_attr",
941 failureStr: formatv(Fmt: "\"op '{0}' attribute '{1}' failed to satisfy constraint: "
942 "'{2}'\"",
943 Vals: op.getOperationName(), Vals&: namedAttr->name,
944 Vals: escapeString(value: matcher.getAsConstraint().getSummary()))
945 .str());
946 }
947
948 // Capture the value
949 auto name = tree.getArgName(index: argIndex);
950 // `$_` is a special symbol to ignore op argument matching.
951 if (!name.empty() && name != "_") {
952 os << formatv(Fmt: "{0} = tblgen_attr;\n", Vals&: name);
953 }
954
955 os.unindent() << "}\n";
956}
957
958void PatternEmitter::emitPropertyMatch(DagNode tree, StringRef castedName,
959 int argIndex, int depth) {
960 Operator &op = tree.getDialectOp(mapper: opMap);
961 auto *namedProp = cast<NamedProperty *>(Val: op.getArg(index: argIndex));
962
963 os << "{\n";
964 os.indent() << formatv(
965 Fmt: "[[maybe_unused]] auto tblgen_prop = {0}.getProperties().{1}();\n",
966 Vals&: castedName, Vals: op.getGetterName(name: namedProp->name));
967
968 auto matcher = tree.getArgAsLeaf(index: argIndex);
969 if (!matcher.isUnspecified()) {
970 if (!matcher.isPropMatcher()) {
971 PrintFatalError(
972 ErrorLoc: loc, Msg: formatv(Fmt: "the {1}-th argument of op '{0}' should be a property",
973 Vals: op.getOperationName(), Vals: argIndex + 1));
974 }
975
976 // If a constraint is specified, we need to generate function call to its
977 // static verifier.
978 StringRef verifier = staticMatcherHelper.getVerifierName(leaf: matcher);
979 emitStaticVerifierCall(
980 funcName: verifier, opName: castedName, arg: "tblgen_prop",
981 failureStr: formatv(Fmt: "\"op '{0}' property '{1}' failed to satisfy constraint: "
982 "'{2}'\"",
983 Vals: op.getOperationName(), Vals&: namedProp->name,
984 Vals: escapeString(value: matcher.getAsConstraint().getSummary()))
985 .str());
986 }
987
988 // Capture the value
989 auto name = tree.getArgName(index: argIndex);
990 // `$_` is a special symbol to ignore op argument matching.
991 if (!name.empty() && name != "_") {
992 os << formatv(Fmt: "{0} = tblgen_prop;\n", Vals&: name);
993 }
994
995 os.unindent() << "}\n";
996}
997
998void PatternEmitter::emitMatchCheck(
999 StringRef opName, const FmtObjectBase &matchFmt,
1000 const llvm::formatv_object_base &failureFmt) {
1001 emitMatchCheck(opName, matchStr: matchFmt.str(), failureStr: failureFmt.str());
1002}
1003
1004void PatternEmitter::emitMatchCheck(StringRef opName,
1005 const std::string &matchStr,
1006 const std::string &failureStr) {
1007
1008 os << "if (!(" << matchStr << "))";
1009 os.scope(open: "{\n", close: "\n}\n").os << "return rewriter.notifyMatchFailure(" << opName
1010 << ", [&](::mlir::Diagnostic &diag) {\n diag << "
1011 << failureStr << ";\n});";
1012}
1013
1014void PatternEmitter::emitMatchLogic(DagNode tree, StringRef opName) {
1015 LLVM_DEBUG(llvm::dbgs() << "--- start emitting match logic ---\n");
1016 int depth = 0;
1017 emitMatch(tree, name: opName, depth);
1018
1019 // Some of the operands could be bound to the same symbol name, we need
1020 // to enforce equality constraint on those.
1021 // This has to happen before user provided constraints, which may assume the
1022 // same name checks are already performed, since in the pattern source code
1023 // the user provided constraints appear later.
1024 // TODO: we should be able to emit equality checks early
1025 // and short circuit unnecessary work if vars are not equal.
1026 for (auto symbolInfoIt = symbolInfoMap.begin();
1027 symbolInfoIt != symbolInfoMap.end();) {
1028 auto range = symbolInfoMap.getRangeOfEqualElements(key: symbolInfoIt->first);
1029 auto startRange = range.first;
1030 auto endRange = range.second;
1031
1032 auto firstOperand = symbolInfoIt->second.getVarName(name: symbolInfoIt->first);
1033 for (++startRange; startRange != endRange; ++startRange) {
1034 auto secondOperand = startRange->second.getVarName(name: symbolInfoIt->first);
1035 emitMatchCheck(
1036 opName,
1037 matchStr: formatv(Fmt: "*{0}.begin() == *{1}.begin()", Vals&: firstOperand, Vals&: secondOperand),
1038 failureStr: formatv(Fmt: "\"Operands '{0}' and '{1}' must be equal\"", Vals&: firstOperand,
1039 Vals&: secondOperand));
1040 }
1041
1042 symbolInfoIt = endRange;
1043 }
1044
1045 for (auto &appliedConstraint : pattern.getConstraints()) {
1046 auto &constraint = appliedConstraint.constraint;
1047 auto &entities = appliedConstraint.entities;
1048
1049 auto condition = constraint.getConditionTemplate();
1050 if (isa<TypeConstraint>(Val: constraint)) {
1051 if (entities.size() != 1)
1052 PrintFatalError(ErrorLoc: loc, Msg: "type constraint requires exactly one argument");
1053
1054 auto self = formatv(Fmt: "({0}.getType())",
1055 Vals: symbolInfoMap.getValueAndRangeUse(symbol: entities.front()));
1056 emitMatchCheck(
1057 opName, matchFmt: tgfmt(fmt: condition, ctx: &fmtCtx.withSelf(subst: self.str())),
1058 failureFmt: formatv(Fmt: "\"value entity '{0}' failed to satisfy constraint: '{1}'\"",
1059 Vals&: entities.front(), Vals: escapeString(value: constraint.getSummary())));
1060
1061 } else if (isa<AttrConstraint>(Val: constraint)) {
1062 PrintFatalError(
1063 ErrorLoc: loc, Msg: "cannot use AttrConstraint in Pattern multi-entity constraints");
1064 } else {
1065 // TODO: replace formatv arguments with the exact specified
1066 // args.
1067 if (entities.size() > 4) {
1068 PrintFatalError(ErrorLoc: loc, Msg: "only support up to 4-entity constraints now");
1069 }
1070 SmallVector<std::string, 4> names;
1071 int i = 0;
1072 for (int e = entities.size(); i < e; ++i)
1073 names.push_back(Elt: symbolInfoMap.getValueAndRangeUse(symbol: entities[i]));
1074 std::string self = appliedConstraint.self;
1075 if (!self.empty())
1076 self = symbolInfoMap.getValueAndRangeUse(symbol: self);
1077 for (; i < 4; ++i)
1078 names.push_back(Elt: "<unused>");
1079 emitMatchCheck(opName,
1080 matchFmt: tgfmt(fmt: condition, ctx: &fmtCtx.withSelf(subst: self), vals&: names[0],
1081 vals&: names[1], vals&: names[2], vals&: names[3]),
1082 failureFmt: formatv(Fmt: "\"entities '{0}' failed to satisfy constraint: "
1083 "'{1}'\"",
1084 Vals: llvm::join(R&: entities, Separator: ", "),
1085 Vals: escapeString(value: constraint.getSummary())));
1086 }
1087 }
1088
1089 LLVM_DEBUG(llvm::dbgs() << "--- done emitting match logic ---\n");
1090}
1091
1092void PatternEmitter::collectOps(DagNode tree,
1093 llvm::SmallPtrSetImpl<const Operator *> &ops) {
1094 // Check if this tree is an operation.
1095 if (tree.isOperation()) {
1096 const Operator &op = tree.getDialectOp(mapper: opMap);
1097 LLVM_DEBUG(llvm::dbgs()
1098 << "found operation " << op.getOperationName() << '\n');
1099 ops.insert(Ptr: &op);
1100 }
1101
1102 // Recurse the arguments of the tree.
1103 for (unsigned i = 0, e = tree.getNumArgs(); i != e; ++i)
1104 if (auto child = tree.getArgAsNestedDag(index: i))
1105 collectOps(tree: child, ops);
1106}
1107
1108void PatternEmitter::emit(StringRef rewriteName) {
1109 // Get the DAG tree for the source pattern.
1110 DagNode sourceTree = pattern.getSourcePattern();
1111
1112 const Operator &rootOp = pattern.getSourceRootOp();
1113 auto rootName = rootOp.getOperationName();
1114
1115 // Collect the set of result operations.
1116 llvm::SmallPtrSet<const Operator *, 4> resultOps;
1117 LLVM_DEBUG(llvm::dbgs() << "start collecting ops used in result patterns\n");
1118 for (unsigned i = 0, e = pattern.getNumResultPatterns(); i != e; ++i) {
1119 collectOps(tree: pattern.getResultPattern(index: i), ops&: resultOps);
1120 }
1121 LLVM_DEBUG(llvm::dbgs() << "done collecting ops used in result patterns\n");
1122
1123 // Emit RewritePattern for Pattern.
1124 auto locs = pattern.getLocation(/*forSourceOutput=*/true);
1125 os << formatv(Fmt: "/* Generated from:\n {0:$[ instantiating\n ]}\n*/\n",
1126 Vals: llvm::reverse(C&: locs));
1127 os << formatv(Fmt: R"(struct {0} : public ::mlir::RewritePattern {
1128 {0}(::mlir::MLIRContext *context)
1129 : ::mlir::RewritePattern("{1}", {2}, context, {{)",
1130 Vals&: rewriteName, Vals&: rootName, Vals: pattern.getBenefit());
1131 // Sort result operators by name.
1132 llvm::SmallVector<const Operator *, 4> sortedResultOps(resultOps.begin(),
1133 resultOps.end());
1134 llvm::sort(C&: sortedResultOps, Comp: [&](const Operator *lhs, const Operator *rhs) {
1135 return lhs->getOperationName() < rhs->getOperationName();
1136 });
1137 llvm::interleaveComma(c: sortedResultOps, os, each_fn: [&](const Operator *op) {
1138 os << '"' << op->getOperationName() << '"';
1139 });
1140 os << "}) {}\n";
1141
1142 // Emit matchAndRewrite() function.
1143 {
1144 auto classScope = os.scope();
1145 os.printReindented(str: R"(
1146 ::llvm::LogicalResult matchAndRewrite(::mlir::Operation *op0,
1147 ::mlir::PatternRewriter &rewriter) const override {)")
1148 << '\n';
1149 {
1150 auto functionScope = os.scope();
1151
1152 // Register all symbols bound in the source pattern.
1153 pattern.collectSourcePatternBoundSymbols(infoMap&: symbolInfoMap);
1154
1155 LLVM_DEBUG(llvm::dbgs()
1156 << "start creating local variables for capturing matches\n");
1157 os << "// Variables for capturing values and attributes used while "
1158 "creating ops\n";
1159 // Create local variables for storing the arguments and results bound
1160 // to symbols.
1161 for (const auto &symbolInfoPair : symbolInfoMap) {
1162 const auto &symbol = symbolInfoPair.first;
1163 const auto &info = symbolInfoPair.second;
1164
1165 os << info.getVarDecl(name: symbol);
1166 }
1167 // TODO: capture ops with consistent numbering so that it can be
1168 // reused for fused loc.
1169 os << "::llvm::SmallVector<::mlir::Operation *, 4> tblgen_ops;\n\n";
1170 LLVM_DEBUG(llvm::dbgs()
1171 << "done creating local variables for capturing matches\n");
1172
1173 os << "// Match\n";
1174 os << "tblgen_ops.push_back(op0);\n";
1175 emitMatchLogic(tree: sourceTree, opName: "op0");
1176
1177 os << "\n// Rewrite\n";
1178 emitRewriteLogic();
1179
1180 os << "return ::mlir::success();\n";
1181 }
1182 os << "}\n";
1183 }
1184 os << "};\n\n";
1185}
1186
1187void PatternEmitter::emitRewriteLogic() {
1188 LLVM_DEBUG(llvm::dbgs() << "--- start emitting rewrite logic ---\n");
1189 const Operator &rootOp = pattern.getSourceRootOp();
1190 int numExpectedResults = rootOp.getNumResults();
1191 int numResultPatterns = pattern.getNumResultPatterns();
1192
1193 // First register all symbols bound to ops generated in result patterns.
1194 pattern.collectResultPatternBoundSymbols(infoMap&: symbolInfoMap);
1195
1196 // Only the last N static values generated are used to replace the matched
1197 // root N-result op. We need to calculate the starting index (of the results
1198 // of the matched op) each result pattern is to replace.
1199 SmallVector<int, 4> offsets(numResultPatterns + 1, numExpectedResults);
1200 // If we don't need to replace any value at all, set the replacement starting
1201 // index as the number of result patterns so we skip all of them when trying
1202 // to replace the matched op's results.
1203 int replStartIndex = numExpectedResults == 0 ? numResultPatterns : -1;
1204 for (int i = numResultPatterns - 1; i >= 0; --i) {
1205 auto numValues = getNodeValueCount(node: pattern.getResultPattern(index: i));
1206 offsets[i] = offsets[i + 1] - numValues;
1207 if (offsets[i] == 0) {
1208 if (replStartIndex == -1)
1209 replStartIndex = i;
1210 } else if (offsets[i] < 0 && offsets[i + 1] > 0) {
1211 auto error = formatv(
1212 Fmt: "cannot use the same multi-result op '{0}' to generate both "
1213 "auxiliary values and values to be used for replacing the matched op",
1214 Vals: pattern.getResultPattern(index: i).getSymbol());
1215 PrintFatalError(ErrorLoc: loc, Msg: error);
1216 }
1217 }
1218
1219 if (offsets.front() > 0) {
1220 const char error[] =
1221 "not enough values generated to replace the matched op";
1222 PrintFatalError(ErrorLoc: loc, Msg: error);
1223 }
1224
1225 os << "auto odsLoc = rewriter.getFusedLoc({";
1226 for (int i = 0, e = pattern.getSourcePattern().getNumOps(); i != e; ++i) {
1227 os << (i ? ", " : "") << "tblgen_ops[" << i << "]->getLoc()";
1228 }
1229 os << "}); (void)odsLoc;\n";
1230
1231 // Process auxiliary result patterns.
1232 for (int i = 0; i < replStartIndex; ++i) {
1233 DagNode resultTree = pattern.getResultPattern(index: i);
1234 auto val = handleResultPattern(resultTree, resultIndex: offsets[i], depth: 0);
1235 // Normal op creation will be streamed to `os` by the above call; but
1236 // NativeCodeCall will only be materialized to `os` if it is used. Here
1237 // we are handling auxiliary patterns so we want the side effect even if
1238 // NativeCodeCall is not replacing matched root op's results.
1239 if (resultTree.isNativeCodeCall() &&
1240 resultTree.getNumReturnsOfNativeCode() == 0)
1241 os << val << ";\n";
1242 }
1243
1244 auto processSupplementalPatterns = [&]() {
1245 int numSupplementalPatterns = pattern.getNumSupplementalPatterns();
1246 for (int i = 0, offset = -numSupplementalPatterns;
1247 i < numSupplementalPatterns; ++i) {
1248 DagNode resultTree = pattern.getSupplementalPattern(index: i);
1249 auto val = handleResultPattern(resultTree, resultIndex: offset++, depth: 0);
1250 if (resultTree.isNativeCodeCall() &&
1251 resultTree.getNumReturnsOfNativeCode() == 0)
1252 os << val << ";\n";
1253 }
1254 };
1255
1256 if (numExpectedResults == 0) {
1257 assert(replStartIndex >= numResultPatterns &&
1258 "invalid auxiliary vs. replacement pattern division!");
1259 processSupplementalPatterns();
1260 // No result to replace. Just erase the op.
1261 os << "rewriter.eraseOp(op0);\n";
1262 } else {
1263 // Process replacement result patterns.
1264 os << "::llvm::SmallVector<::mlir::Value, 4> tblgen_repl_values;\n";
1265 for (int i = replStartIndex; i < numResultPatterns; ++i) {
1266 DagNode resultTree = pattern.getResultPattern(index: i);
1267 auto val = handleResultPattern(resultTree, resultIndex: offsets[i], depth: 0);
1268 os << "\n";
1269 // Resolve each symbol for all range use so that we can loop over them.
1270 // We need an explicit cast to `SmallVector` to capture the cases where
1271 // `{0}` resolves to an `Operation::result_range` as well as cases that
1272 // are not iterable (e.g. vector that gets wrapped in additional braces by
1273 // RewriterGen).
1274 // TODO: Revisit the need for materializing a vector.
1275 os << symbolInfoMap.getAllRangeUse(
1276 symbol: val,
1277 fmt: "for (auto v: ::llvm::SmallVector<::mlir::Value, 4>{ {0} }) {{\n"
1278 " tblgen_repl_values.push_back(v);\n}\n",
1279 separator: "\n");
1280 }
1281 processSupplementalPatterns();
1282 os << "\nrewriter.replaceOp(op0, tblgen_repl_values);\n";
1283 }
1284
1285 LLVM_DEBUG(llvm::dbgs() << "--- done emitting rewrite logic ---\n");
1286}
1287
1288std::string PatternEmitter::getUniqueSymbol(const Operator *op) {
1289 return std::string(
1290 formatv(Fmt: "tblgen_{0}_{1}", Vals: op->getCppClassName(), Vals: nextValueId++));
1291}
1292
1293std::string PatternEmitter::handleResultPattern(DagNode resultTree,
1294 int resultIndex, int depth) {
1295 LLVM_DEBUG(llvm::dbgs() << "handle result pattern: ");
1296 LLVM_DEBUG(resultTree.print(llvm::dbgs()));
1297 LLVM_DEBUG(llvm::dbgs() << '\n');
1298
1299 if (resultTree.isLocationDirective()) {
1300 PrintFatalError(ErrorLoc: loc,
1301 Msg: "location directive can only be used with op creation");
1302 }
1303
1304 if (resultTree.isNativeCodeCall())
1305 return handleReplaceWithNativeCodeCall(resultTree, depth);
1306
1307 if (resultTree.isReplaceWithValue())
1308 return handleReplaceWithValue(tree: resultTree).str();
1309
1310 if (resultTree.isVariadic())
1311 return handleVariadic(tree: resultTree, depth);
1312
1313 // Normal op creation.
1314 auto symbol = handleOpCreation(tree: resultTree, resultIndex, depth);
1315 if (resultTree.getSymbol().empty()) {
1316 // This is an op not explicitly bound to a symbol in the rewrite rule.
1317 // Register the auto-generated symbol for it.
1318 symbolInfoMap.bindOpResult(symbol, op: pattern.getDialectOp(node: resultTree));
1319 }
1320 return symbol;
1321}
1322
1323std::string PatternEmitter::handleVariadic(DagNode tree, int depth) {
1324 assert(tree.isVariadic());
1325
1326 std::string output;
1327 llvm::raw_string_ostream oss(output);
1328 auto name = std::string(formatv(Fmt: "tblgen_variadic_values_{0}", Vals: nextValueId++));
1329 symbolInfoMap.bindValue(symbol: name);
1330 oss << "::llvm::SmallVector<::mlir::Value, 4> " << name << ";\n";
1331 for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
1332 if (auto child = tree.getArgAsNestedDag(index: i)) {
1333 oss << name << ".push_back(" << handleResultPattern(resultTree: child, resultIndex: i, depth: depth + 1)
1334 << ");\n";
1335 } else {
1336 oss << name << ".push_back("
1337 << handleOpArgument(leaf: tree.getArgAsLeaf(index: i), patArgName: tree.getArgName(index: i))
1338 << ");\n";
1339 }
1340 }
1341
1342 os << oss.str();
1343 return name;
1344}
1345
1346StringRef PatternEmitter::handleReplaceWithValue(DagNode tree) {
1347 assert(tree.isReplaceWithValue());
1348
1349 if (tree.getNumArgs() != 1) {
1350 PrintFatalError(
1351 ErrorLoc: loc, Msg: "replaceWithValue directive must take exactly one argument");
1352 }
1353
1354 if (!tree.getSymbol().empty()) {
1355 PrintFatalError(ErrorLoc: loc, Msg: "cannot bind symbol to replaceWithValue");
1356 }
1357
1358 return tree.getArgName(index: 0);
1359}
1360
1361std::string PatternEmitter::handleLocationDirective(DagNode tree) {
1362 assert(tree.isLocationDirective());
1363 auto lookUpArgLoc = [this, &tree](int idx) {
1364 const auto *const lookupFmt = "{0}.getLoc()";
1365 return symbolInfoMap.getValueAndRangeUse(symbol: tree.getArgName(index: idx), fmt: lookupFmt);
1366 };
1367
1368 if (tree.getNumArgs() == 0)
1369 llvm::PrintFatalError(
1370 Msg: "At least one argument to location directive required");
1371
1372 if (!tree.getSymbol().empty())
1373 PrintFatalError(ErrorLoc: loc, Msg: "cannot bind symbol to location");
1374
1375 if (tree.getNumArgs() == 1) {
1376 DagLeaf leaf = tree.getArgAsLeaf(index: 0);
1377 if (leaf.isStringAttr())
1378 return formatv(Fmt: "::mlir::NameLoc::get(rewriter.getStringAttr(\"{0}\"))",
1379 Vals: leaf.getStringAttr())
1380 .str();
1381 return lookUpArgLoc(0);
1382 }
1383
1384 std::string ret;
1385 llvm::raw_string_ostream os(ret);
1386 std::string strAttr;
1387 os << "rewriter.getFusedLoc({";
1388 bool first = true;
1389 for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
1390 DagLeaf leaf = tree.getArgAsLeaf(index: i);
1391 // Handle the optional string value.
1392 if (leaf.isStringAttr()) {
1393 if (!strAttr.empty())
1394 llvm::PrintFatalError(Msg: "Only one string attribute may be specified");
1395 strAttr = leaf.getStringAttr();
1396 continue;
1397 }
1398 os << (first ? "" : ", ") << lookUpArgLoc(i);
1399 first = false;
1400 }
1401 os << "}";
1402 if (!strAttr.empty()) {
1403 os << ", rewriter.getStringAttr(\"" << strAttr << "\")";
1404 }
1405 os << ")";
1406 return os.str();
1407}
1408
1409std::string PatternEmitter::handleReturnTypeArg(DagNode returnType, int i,
1410 int depth) {
1411 // Nested NativeCodeCall.
1412 if (auto dagNode = returnType.getArgAsNestedDag(index: i)) {
1413 if (!dagNode.isNativeCodeCall())
1414 PrintFatalError(ErrorLoc: loc, Msg: "nested DAG in `returnType` must be a native code "
1415 "call");
1416 return handleReplaceWithNativeCodeCall(resultTree: dagNode, depth);
1417 }
1418 // String literal.
1419 auto dagLeaf = returnType.getArgAsLeaf(index: i);
1420 if (dagLeaf.isStringAttr())
1421 return tgfmt(fmt: dagLeaf.getStringAttr(), ctx: &fmtCtx);
1422 return tgfmt(
1423 fmt: "$0.getType()", ctx: &fmtCtx,
1424 vals: handleOpArgument(leaf: returnType.getArgAsLeaf(index: i), patArgName: returnType.getArgName(index: i)));
1425}
1426
1427std::string PatternEmitter::handleOpArgument(DagLeaf leaf,
1428 StringRef patArgName) {
1429 if (leaf.isStringAttr())
1430 PrintFatalError(ErrorLoc: loc, Msg: "raw string not supported as argument");
1431 if (leaf.isConstantAttr()) {
1432 auto constAttr = leaf.getAsConstantAttr();
1433 return handleConstantAttr(attr: constAttr.getAttribute(),
1434 value: constAttr.getConstantValue());
1435 }
1436 if (leaf.isEnumCase()) {
1437 auto enumCase = leaf.getAsEnumCase();
1438 // This is an enum case backed by an IntegerAttr. We need to get its value
1439 // to build the constant.
1440 std::string val = std::to_string(val: enumCase.getValue());
1441 return handleConstantAttr(attr: Attribute(&enumCase.getDef()), value: val);
1442 }
1443 if (leaf.isConstantProp()) {
1444 auto constantProp = leaf.getAsConstantProp();
1445 return constantProp.getValue().str();
1446 }
1447
1448 LLVM_DEBUG(llvm::dbgs() << "handle argument '" << patArgName << "'\n");
1449 auto argName = symbolInfoMap.getValueAndRangeUse(symbol: patArgName);
1450 if (leaf.isUnspecified() || leaf.isOperandMatcher()) {
1451 LLVM_DEBUG(llvm::dbgs() << "replace " << patArgName << " with '" << argName
1452 << "' (via symbol ref)\n");
1453 return argName;
1454 }
1455 if (leaf.isNativeCodeCall()) {
1456 auto repl = tgfmt(fmt: leaf.getNativeCodeTemplate(), ctx: &fmtCtx.withSelf(subst: argName));
1457 LLVM_DEBUG(llvm::dbgs() << "replace " << patArgName << " with '" << repl
1458 << "' (via NativeCodeCall)\n");
1459 return std::string(repl);
1460 }
1461 PrintFatalError(ErrorLoc: loc, Msg: "unhandled case when rewriting op");
1462}
1463
1464std::string PatternEmitter::handleReplaceWithNativeCodeCall(DagNode tree,
1465 int depth) {
1466 LLVM_DEBUG(llvm::dbgs() << "handle NativeCodeCall pattern: ");
1467 LLVM_DEBUG(tree.print(llvm::dbgs()));
1468 LLVM_DEBUG(llvm::dbgs() << '\n');
1469
1470 auto fmt = tree.getNativeCodeTemplate();
1471
1472 SmallVector<std::string, 16> attrs;
1473
1474 auto tail = getTrailingDirectives(tree);
1475 if (tail.returnType)
1476 PrintFatalError(ErrorLoc: loc, Msg: "`NativeCodeCall` cannot have return type specifier");
1477 auto locToUse = getLocation(tail);
1478
1479 for (int i = 0, e = tree.getNumArgs() - tail.numDirectives; i != e; ++i) {
1480 if (tree.isNestedDagArg(index: i)) {
1481 attrs.push_back(
1482 Elt: handleResultPattern(resultTree: tree.getArgAsNestedDag(index: i), resultIndex: i, depth: depth + 1));
1483 } else {
1484 attrs.push_back(
1485 Elt: handleOpArgument(leaf: tree.getArgAsLeaf(index: i), patArgName: tree.getArgName(index: i)));
1486 }
1487 LLVM_DEBUG(llvm::dbgs() << "NativeCodeCall argument #" << i
1488 << " replacement: " << attrs[i] << "\n");
1489 }
1490
1491 std::string symbol = tgfmtv(fmt, ctx: &fmtCtx.addSubst(placeholder: "_loc", subst: locToUse),
1492 params: static_cast<ArrayRef<std::string>>(attrs));
1493
1494 // In general, NativeCodeCall without naming binding don't need this. To
1495 // ensure void helper function has been correctly labeled, i.e., use
1496 // NativeCodeCallVoid, we cache the result to a local variable so that we will
1497 // get a compilation error in the auto-generated file.
1498 // Example.
1499 // // In the td file
1500 // Pat<(...), (NativeCodeCall<Foo> ...)>
1501 //
1502 // ---
1503 //
1504 // // In the auto-generated .cpp
1505 // ...
1506 // // Causes compilation error if Foo() returns void.
1507 // auto nativeVar = Foo();
1508 // ...
1509 if (tree.getNumReturnsOfNativeCode() != 0) {
1510 // Determine the local variable name for return value.
1511 std::string varName =
1512 SymbolInfoMap::getValuePackName(symbol: tree.getSymbol()).str();
1513 if (varName.empty()) {
1514 varName = formatv(Fmt: "nativeVar_{0}", Vals: nextValueId++);
1515 // Register the local variable for later uses.
1516 symbolInfoMap.bindValues(symbol: varName, numValues: tree.getNumReturnsOfNativeCode());
1517 }
1518
1519 // Catch the return value of helper function.
1520 os << formatv(Fmt: "auto {0} = {1}; (void){0};\n", Vals&: varName, Vals&: symbol);
1521
1522 if (!tree.getSymbol().empty())
1523 symbol = tree.getSymbol().str();
1524 else
1525 symbol = varName;
1526 }
1527
1528 return symbol;
1529}
1530
1531int PatternEmitter::getNodeValueCount(DagNode node) {
1532 if (node.isOperation()) {
1533 // If the op is bound to a symbol in the rewrite rule, query its result
1534 // count from the symbol info map.
1535 auto symbol = node.getSymbol();
1536 if (!symbol.empty()) {
1537 return symbolInfoMap.getStaticValueCount(symbol);
1538 }
1539 // Otherwise this is an unbound op; we will use all its results.
1540 return pattern.getDialectOp(node).getNumResults();
1541 }
1542
1543 if (node.isNativeCodeCall())
1544 return node.getNumReturnsOfNativeCode();
1545
1546 return 1;
1547}
1548
1549PatternEmitter::TrailingDirectives
1550PatternEmitter::getTrailingDirectives(DagNode tree) {
1551 TrailingDirectives tail = {.location: DagNode(nullptr), .returnType: DagNode(nullptr), .numDirectives: 0};
1552
1553 // Look backwards through the arguments.
1554 auto numPatArgs = tree.getNumArgs();
1555 for (int i = numPatArgs - 1; i >= 0; --i) {
1556 auto dagArg = tree.getArgAsNestedDag(index: i);
1557 // A leaf is not a directive. Stop looking.
1558 if (!dagArg)
1559 break;
1560
1561 auto isLocation = dagArg.isLocationDirective();
1562 auto isReturnType = dagArg.isReturnTypeDirective();
1563 // If encountered a DAG node that isn't a trailing directive, stop looking.
1564 if (!(isLocation || isReturnType))
1565 break;
1566 // Save the directive, but error if one of the same type was already
1567 // found.
1568 ++tail.numDirectives;
1569 if (isLocation) {
1570 if (tail.location)
1571 PrintFatalError(ErrorLoc: loc, Msg: "`location` directive can only be specified "
1572 "once");
1573 tail.location = dagArg;
1574 } else if (isReturnType) {
1575 if (tail.returnType)
1576 PrintFatalError(ErrorLoc: loc, Msg: "`returnType` directive can only be specified "
1577 "once");
1578 tail.returnType = dagArg;
1579 }
1580 }
1581
1582 return tail;
1583}
1584
1585std::string
1586PatternEmitter::getLocation(PatternEmitter::TrailingDirectives &tail) {
1587 if (tail.location)
1588 return handleLocationDirective(tree: tail.location);
1589
1590 // If no explicit location is given, use the default, all fused, location.
1591 return "odsLoc";
1592}
1593
1594std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex,
1595 int depth) {
1596 LLVM_DEBUG(llvm::dbgs() << "create op for pattern: ");
1597 LLVM_DEBUG(tree.print(llvm::dbgs()));
1598 LLVM_DEBUG(llvm::dbgs() << '\n');
1599
1600 Operator &resultOp = tree.getDialectOp(mapper: opMap);
1601 auto numOpArgs = resultOp.getNumArgs();
1602 auto numPatArgs = tree.getNumArgs();
1603
1604 auto tail = getTrailingDirectives(tree);
1605 auto locToUse = getLocation(tail);
1606
1607 auto inPattern = numPatArgs - tail.numDirectives;
1608 if (numOpArgs != inPattern) {
1609 PrintFatalError(ErrorLoc: loc,
1610 Msg: formatv(Fmt: "resultant op '{0}' argument number mismatch: "
1611 "{1} in pattern vs. {2} in definition",
1612 Vals: resultOp.getOperationName(), Vals&: inPattern, Vals&: numOpArgs));
1613 }
1614
1615 // A map to collect all nested DAG child nodes' names, with operand index as
1616 // the key. This includes both bound and unbound child nodes.
1617 ChildNodeIndexNameMap childNodeNames;
1618
1619 // If the argument is a type constraint, then its an operand. Check if the
1620 // op's argument is variadic that the argument in the pattern is too.
1621 auto checkIfMatchedVariadic = [&](int i) {
1622 // FIXME: This does not yet check for variable/leaf case.
1623 // FIXME: Change so that native code call can be handled.
1624 const auto *operand =
1625 llvm::dyn_cast_if_present<NamedTypeConstraint *>(Val: resultOp.getArg(index: i));
1626 if (!operand || !operand->isVariadic())
1627 return;
1628
1629 auto child = tree.getArgAsNestedDag(index: i);
1630 if (!child)
1631 return;
1632
1633 // Skip over replaceWithValues.
1634 while (child.isReplaceWithValue()) {
1635 if (!(child = child.getArgAsNestedDag(index: 0)))
1636 return;
1637 }
1638 if (!child.isNativeCodeCall() && !child.isVariadic())
1639 PrintFatalError(ErrorLoc: loc, Msg: formatv(Fmt: "op expects variadic operand `{0}`, while "
1640 "provided is non-variadic",
1641 Vals: resultOp.getArgName(index: i)));
1642 };
1643
1644 // First go through all the child nodes who are nested DAG constructs to
1645 // create ops for them and remember the symbol names for them, so that we can
1646 // use the results in the current node. This happens in a recursive manner.
1647 for (int i = 0, e = tree.getNumArgs() - tail.numDirectives; i != e; ++i) {
1648 checkIfMatchedVariadic(i);
1649 if (auto child = tree.getArgAsNestedDag(index: i))
1650 childNodeNames[i] = handleResultPattern(resultTree: child, resultIndex: i, depth: depth + 1);
1651 }
1652
1653 // The name of the local variable holding this op.
1654 std::string valuePackName;
1655 // The symbol for holding the result of this pattern. Note that the result of
1656 // this pattern is not necessarily the same as the variable created by this
1657 // pattern because we can use `__N` suffix to refer only a specific result if
1658 // the generated op is a multi-result op.
1659 std::string resultValue;
1660 if (tree.getSymbol().empty()) {
1661 // No symbol is explicitly bound to this op in the pattern. Generate a
1662 // unique name.
1663 valuePackName = resultValue = getUniqueSymbol(op: &resultOp);
1664 } else {
1665 resultValue = std::string(tree.getSymbol());
1666 // Strip the index to get the name for the value pack and use it to name the
1667 // local variable for the op.
1668 valuePackName = std::string(SymbolInfoMap::getValuePackName(symbol: resultValue));
1669 }
1670
1671 // Create the local variable for this op.
1672 os << formatv(Fmt: "{0} {1};\n{{\n", Vals: resultOp.getQualCppClassName(),
1673 Vals&: valuePackName);
1674
1675 // Right now ODS don't have general type inference support. Except a few
1676 // special cases listed below, DRR needs to supply types for all results
1677 // when building an op.
1678 bool isSameOperandsAndResultType =
1679 resultOp.getTrait(trait: "::mlir::OpTrait::SameOperandsAndResultType");
1680 bool useFirstAttr =
1681 resultOp.getTrait(trait: "::mlir::OpTrait::FirstAttrDerivedResultType");
1682
1683 if (!tail.returnType && (isSameOperandsAndResultType || useFirstAttr)) {
1684 // We know how to deduce the result type for ops with these traits and we've
1685 // generated builders taking aggregate parameters. Use those builders to
1686 // create the ops.
1687
1688 // First prepare local variables for op arguments used in builder call.
1689 createAggregateLocalVarsForOpArgs(node: tree, childNodeNames, depth);
1690
1691 // Then create the op.
1692 os.scope(open: "", close: "\n}\n").os << formatv(
1693 Fmt: "{0} = {1}::create(rewriter, {2}, tblgen_values, {3});", Vals&: valuePackName,
1694 Vals: resultOp.getQualCppClassName(), Vals&: locToUse, Vals: "tblgen_props");
1695 return resultValue;
1696 }
1697
1698 bool usePartialResults = valuePackName != resultValue;
1699
1700 if (!tail.returnType && (usePartialResults || depth > 0 || resultIndex < 0)) {
1701 // For these cases (broadcastable ops, op results used both as auxiliary
1702 // values and replacement values, ops in nested patterns, auxiliary ops), we
1703 // still need to supply the result types when building the op. But because
1704 // we don't generate a builder automatically with ODS for them, it's the
1705 // developer's responsibility to make sure such a builder (with result type
1706 // deduction ability) exists. We go through the separate-parameter builder
1707 // here given that it's easier for developers to write compared to
1708 // aggregate-parameter builders.
1709 createSeparateLocalVarsForOpArgs(node: tree, childNodeNames);
1710
1711 os.scope().os << formatv(Fmt: "{0} = {1}::create(rewriter, {2}", Vals&: valuePackName,
1712 Vals: resultOp.getQualCppClassName(), Vals&: locToUse);
1713 supplyValuesForOpArgs(node: tree, childNodeNames, depth);
1714 os << "\n );\n}\n";
1715 return resultValue;
1716 }
1717
1718 // If we are provided explicit return types, use them to build the op.
1719 // However, if depth == 0 and resultIndex >= 0, it means we are replacing
1720 // the values generated from the source pattern root op. Then we must use the
1721 // source pattern's value types to determine the value type of the generated
1722 // op here.
1723 if (depth == 0 && resultIndex >= 0 && tail.returnType)
1724 PrintFatalError(ErrorLoc: loc, Msg: "Cannot specify explicit return types in an op whose "
1725 "return values replace the source pattern's root op");
1726
1727 // First prepare local variables for op arguments used in builder call.
1728 createAggregateLocalVarsForOpArgs(node: tree, childNodeNames, depth);
1729
1730 // Then prepare the result types. We need to specify the types for all
1731 // results.
1732 os.indent() << formatv(Fmt: "::llvm::SmallVector<::mlir::Type, 4> tblgen_types; "
1733 "(void)tblgen_types;\n");
1734 int numResults = resultOp.getNumResults();
1735 if (tail.returnType) {
1736 auto numRetTys = tail.returnType.getNumArgs();
1737 for (int i = 0; i < numRetTys; ++i) {
1738 auto varName = handleReturnTypeArg(returnType: tail.returnType, i, depth: depth + 1);
1739 os << "tblgen_types.push_back(" << varName << ");\n";
1740 }
1741 } else {
1742 if (numResults != 0) {
1743 // Copy the result types from the source pattern.
1744 for (int i = 0; i < numResults; ++i)
1745 os << formatv(Fmt: "for (auto v: castedOp0.getODSResults({0})) {{\n"
1746 " tblgen_types.push_back(v.getType());\n}\n",
1747 Vals: resultIndex + i);
1748 }
1749 }
1750 os << formatv(Fmt: "{0} = {1}::create(rewriter, {2}, tblgen_types, "
1751 "tblgen_values, {3});\n",
1752 Vals&: valuePackName, Vals: resultOp.getQualCppClassName(), Vals&: locToUse,
1753 Vals: "tblgen_props");
1754 os.unindent() << "}\n";
1755 return resultValue;
1756}
1757
1758void PatternEmitter::createSeparateLocalVarsForOpArgs(
1759 DagNode node, ChildNodeIndexNameMap &childNodeNames) {
1760 Operator &resultOp = node.getDialectOp(mapper: opMap);
1761
1762 // Now prepare operands used for building this op:
1763 // * If the operand is non-variadic, we create a `Value` local variable.
1764 // * If the operand is variadic, we create a `SmallVector<Value>` local
1765 // variable.
1766
1767 int valueIndex = 0; // An index for uniquing local variable names.
1768 for (int argIndex = 0, e = resultOp.getNumArgs(); argIndex < e; ++argIndex) {
1769 const auto *operand = llvm::dyn_cast_if_present<NamedTypeConstraint *>(
1770 Val: resultOp.getArg(index: argIndex));
1771 // We do not need special handling for attributes or properties.
1772 if (!operand)
1773 continue;
1774
1775 raw_indented_ostream::DelimitedScope scope(os);
1776 std::string varName;
1777 if (operand->isVariadic()) {
1778 varName = std::string(formatv(Fmt: "tblgen_values_{0}", Vals: valueIndex++));
1779 os << formatv(Fmt: "::llvm::SmallVector<::mlir::Value, 4> {0};\n", Vals&: varName);
1780 std::string range;
1781 if (node.isNestedDagArg(index: argIndex)) {
1782 range = childNodeNames[argIndex];
1783 } else {
1784 range = std::string(node.getArgName(index: argIndex));
1785 }
1786 // Resolve the symbol for all range use so that we have a uniform way of
1787 // capturing the values.
1788 range = symbolInfoMap.getValueAndRangeUse(symbol: range);
1789 os << formatv(Fmt: "for (auto v: {0}) {{\n {1}.push_back(v);\n}\n", Vals&: range,
1790 Vals&: varName);
1791 } else {
1792 varName = std::string(formatv(Fmt: "tblgen_value_{0}", Vals: valueIndex++));
1793 os << formatv(Fmt: "::mlir::Value {0} = ", Vals&: varName);
1794 if (node.isNestedDagArg(index: argIndex)) {
1795 os << symbolInfoMap.getValueAndRangeUse(symbol: childNodeNames[argIndex]);
1796 } else {
1797 DagLeaf leaf = node.getArgAsLeaf(index: argIndex);
1798 auto symbol =
1799 symbolInfoMap.getValueAndRangeUse(symbol: node.getArgName(index: argIndex));
1800 if (leaf.isNativeCodeCall()) {
1801 os << std::string(
1802 tgfmt(fmt: leaf.getNativeCodeTemplate(), ctx: &fmtCtx.withSelf(subst: symbol)));
1803 } else {
1804 os << symbol;
1805 }
1806 }
1807 os << ";\n";
1808 }
1809
1810 // Update to use the newly created local variable for building the op later.
1811 childNodeNames[argIndex] = varName;
1812 }
1813}
1814
1815void PatternEmitter::supplyValuesForOpArgs(
1816 DagNode node, const ChildNodeIndexNameMap &childNodeNames, int depth) {
1817 Operator &resultOp = node.getDialectOp(mapper: opMap);
1818 for (int argIndex = 0, numOpArgs = resultOp.getNumArgs();
1819 argIndex != numOpArgs; ++argIndex) {
1820 // Start each argument on its own line.
1821 os << ",\n ";
1822
1823 Argument opArg = resultOp.getArg(index: argIndex);
1824 // Handle the case of operand first.
1825 if (auto *operand =
1826 llvm::dyn_cast_if_present<NamedTypeConstraint *>(Val&: opArg)) {
1827 if (!operand->name.empty())
1828 os << "/*" << operand->name << "=*/";
1829 os << childNodeNames.lookup(Val: argIndex);
1830 continue;
1831 }
1832
1833 // The argument in the op definition.
1834 auto opArgName = resultOp.getArgName(index: argIndex);
1835 if (auto subTree = node.getArgAsNestedDag(index: argIndex)) {
1836 if (!subTree.isNativeCodeCall())
1837 PrintFatalError(ErrorLoc: loc, Msg: "only NativeCodeCall allowed in nested dag node "
1838 "for creating attributes and properties");
1839 os << formatv(Fmt: "/*{0}=*/{1}", Vals&: opArgName, Vals: childNodeNames.lookup(Val: argIndex));
1840 } else {
1841 auto leaf = node.getArgAsLeaf(index: argIndex);
1842 // The argument in the result DAG pattern.
1843 auto patArgName = node.getArgName(index: argIndex);
1844 if (leaf.isConstantAttr() || leaf.isEnumCase()) {
1845 // TODO: Refactor out into map to avoid recomputing these.
1846 if (!isa<NamedAttribute *>(Val: opArg))
1847 PrintFatalError(ErrorLoc: loc, Msg: Twine("expected attribute ") + Twine(argIndex));
1848 if (!patArgName.empty())
1849 os << "/*" << patArgName << "=*/";
1850 } else if (leaf.isConstantProp()) {
1851 if (!isa<NamedProperty *>(Val: opArg))
1852 PrintFatalError(ErrorLoc: loc, Msg: Twine("expected property ") + Twine(argIndex));
1853 if (!patArgName.empty())
1854 os << "/*" << patArgName << "=*/";
1855 } else {
1856 os << "/*" << opArgName << "=*/";
1857 }
1858 os << handleOpArgument(leaf, patArgName);
1859 }
1860 }
1861}
1862
1863void PatternEmitter::createAggregateLocalVarsForOpArgs(
1864 DagNode node, const ChildNodeIndexNameMap &childNodeNames, int depth) {
1865 Operator &resultOp = node.getDialectOp(mapper: opMap);
1866
1867 auto scope = os.scope();
1868 os << formatv(Fmt: "::llvm::SmallVector<::mlir::Value, 4> "
1869 "tblgen_values; (void)tblgen_values;\n");
1870 os << formatv(Fmt: "{0}::Properties tblgen_props; (void)tblgen_props;\n",
1871 Vals: resultOp.getQualCppClassName());
1872
1873 const char *setterCmd =
1874 "tblgen_props.{0} = "
1875 "::llvm::dyn_cast_if_present<decltype(tblgen_props.{0})>({1});\n";
1876 const char *propSetterCmd = "tblgen_props.{0}({1});\n";
1877
1878 int numVariadic = 0;
1879 bool hasOperandSegmentSizes = false;
1880 std::vector<std::string> sizes;
1881 for (int argIndex = 0, e = resultOp.getNumArgs(); argIndex < e; ++argIndex) {
1882 if (isa<NamedAttribute *>(Val: resultOp.getArg(index: argIndex))) {
1883 // The argument in the op definition.
1884 auto opArgName = resultOp.getArgName(index: argIndex);
1885 hasOperandSegmentSizes =
1886 hasOperandSegmentSizes || opArgName == "operandSegmentSizes";
1887 if (auto subTree = node.getArgAsNestedDag(index: argIndex)) {
1888 if (!subTree.isNativeCodeCall())
1889 PrintFatalError(ErrorLoc: loc, Msg: "only NativeCodeCall allowed in nested dag node "
1890 "for creating attribute");
1891
1892 os << formatv(Fmt: setterCmd, Vals&: opArgName, Vals: childNodeNames.lookup(Val: argIndex));
1893 } else {
1894 auto leaf = node.getArgAsLeaf(index: argIndex);
1895 // The argument in the result DAG pattern.
1896 auto patArgName = node.getArgName(index: argIndex);
1897 os << formatv(Fmt: setterCmd, Vals&: opArgName, Vals: handleOpArgument(leaf, patArgName));
1898 }
1899 continue;
1900 }
1901
1902 if (isa<NamedProperty *>(Val: resultOp.getArg(index: argIndex))) {
1903 // The argument in the op definition.
1904 auto opArgName = resultOp.getArgName(index: argIndex);
1905 auto setterName = resultOp.getSetterName(name: opArgName);
1906 if (auto subTree = node.getArgAsNestedDag(index: argIndex)) {
1907 if (!subTree.isNativeCodeCall())
1908 PrintFatalError(ErrorLoc: loc, Msg: "only NativeCodeCall allowed in nested dag node "
1909 "for creating property");
1910
1911 os << formatv(Fmt: propSetterCmd, Vals&: setterName,
1912 Vals: childNodeNames.lookup(Val: argIndex));
1913 } else {
1914 auto leaf = node.getArgAsLeaf(index: argIndex);
1915 // The argument in the result DAG pattern.
1916 auto patArgName = node.getArgName(index: argIndex);
1917 // The argument in the result DAG pattern.
1918 os << formatv(Fmt: propSetterCmd, Vals&: setterName,
1919 Vals: handleOpArgument(leaf, patArgName));
1920 }
1921 continue;
1922 }
1923
1924 const auto *operand =
1925 cast<NamedTypeConstraint *>(Val: resultOp.getArg(index: argIndex));
1926 if (operand->isVariadic()) {
1927 ++numVariadic;
1928 std::string range;
1929 if (node.isNestedDagArg(index: argIndex)) {
1930 range = childNodeNames.lookup(Val: argIndex);
1931 } else {
1932 range = std::string(node.getArgName(index: argIndex));
1933 }
1934 // Resolve the symbol for all range use so that we have a uniform way of
1935 // capturing the values.
1936 range = symbolInfoMap.getValueAndRangeUse(symbol: range);
1937 os << formatv(Fmt: "for (auto v: {0}) {{\n tblgen_values.push_back(v);\n}\n",
1938 Vals&: range);
1939 sizes.push_back(x: formatv(Fmt: "static_cast<int32_t>({0}.size())", Vals&: range));
1940 } else {
1941 sizes.emplace_back(args: "1");
1942 os << formatv(Fmt: "tblgen_values.push_back(");
1943 if (node.isNestedDagArg(index: argIndex)) {
1944 os << symbolInfoMap.getValueAndRangeUse(
1945 symbol: childNodeNames.lookup(Val: argIndex));
1946 } else {
1947 DagLeaf leaf = node.getArgAsLeaf(index: argIndex);
1948 if (leaf.isConstantAttr())
1949 // TODO: Use better location
1950 PrintFatalError(
1951 ErrorLoc: loc,
1952 Msg: "attribute found where value was expected, if attempting to use "
1953 "constant value, construct a constant op with given attribute "
1954 "instead");
1955
1956 auto symbol =
1957 symbolInfoMap.getValueAndRangeUse(symbol: node.getArgName(index: argIndex));
1958 if (leaf.isNativeCodeCall()) {
1959 os << std::string(
1960 tgfmt(fmt: leaf.getNativeCodeTemplate(), ctx: &fmtCtx.withSelf(subst: symbol)));
1961 } else {
1962 os << symbol;
1963 }
1964 }
1965 os << ");\n";
1966 }
1967 }
1968
1969 if (numVariadic > 1 && !hasOperandSegmentSizes) {
1970 // Only set size if it can't be computed.
1971 const auto *sameVariadicSize =
1972 resultOp.getTrait(trait: "::mlir::OpTrait::SameVariadicOperandSize");
1973 if (!sameVariadicSize) {
1974 const char *setSizes = R"(
1975 tblgen_props.operandSegmentSizes = {{ {0} };
1976 )";
1977 os.printReindented(str: formatv(Fmt: setSizes, Vals: llvm::join(R&: sizes, Separator: ", ")).str());
1978 }
1979 }
1980}
1981
1982StaticMatcherHelper::StaticMatcherHelper(raw_ostream &os,
1983 const RecordKeeper &records,
1984 RecordOperatorMap &mapper)
1985 : opMap(mapper), staticVerifierEmitter(os, records) {}
1986
1987void StaticMatcherHelper::populateStaticMatchers(raw_ostream &os) {
1988 // PatternEmitter will use the static matcher if there's one generated. To
1989 // ensure that all the dependent static matchers are generated before emitting
1990 // the matching logic of the DagNode, we use topological order to achieve it.
1991 for (auto &dagInfo : topologicalOrder) {
1992 DagNode node = dagInfo.first;
1993 if (!useStaticMatcher(node))
1994 continue;
1995
1996 std::string funcName =
1997 formatv(Fmt: "static_dag_matcher_{0}", Vals: staticMatcherCounter++);
1998 assert(!matcherNames.contains(node));
1999 PatternEmitter(dagInfo.second, &opMap, os, *this)
2000 .emitStaticMatcher(tree: node, funcName);
2001 matcherNames[node] = funcName;
2002 }
2003}
2004
2005void StaticMatcherHelper::populateStaticConstraintFunctions(raw_ostream &os) {
2006 staticVerifierEmitter.emitPatternConstraints(constraints: constraints.getArrayRef());
2007}
2008
2009void StaticMatcherHelper::addPattern(const Record *record) {
2010 Pattern pat(record, &opMap);
2011
2012 // While generating the function body of the DAG matcher, it may depends on
2013 // other DAG matchers. To ensure the dependent matchers are ready, we compute
2014 // the topological order for all the DAGs and emit the DAG matchers in this
2015 // order.
2016 llvm::unique_function<void(DagNode)> dfs = [&](DagNode node) {
2017 ++refStats[node];
2018
2019 if (refStats[node] != 1)
2020 return;
2021
2022 for (unsigned i = 0, e = node.getNumArgs(); i < e; ++i)
2023 if (DagNode sibling = node.getArgAsNestedDag(index: i))
2024 dfs(sibling);
2025 else {
2026 DagLeaf leaf = node.getArgAsLeaf(index: i);
2027 if (!leaf.isUnspecified())
2028 constraints.insert(X: leaf);
2029 }
2030
2031 topologicalOrder.push_back(Elt: std::make_pair(x&: node, y&: record));
2032 };
2033
2034 dfs(pat.getSourcePattern());
2035}
2036
2037StringRef StaticMatcherHelper::getVerifierName(DagLeaf leaf) {
2038 if (leaf.isAttrMatcher()) {
2039 std::optional<StringRef> constraint =
2040 staticVerifierEmitter.getAttrConstraintFn(constraint: leaf.getAsConstraint());
2041 assert(constraint && "attribute constraint was not uniqued");
2042 return *constraint;
2043 }
2044 if (leaf.isPropMatcher()) {
2045 std::optional<StringRef> constraint =
2046 staticVerifierEmitter.getPropConstraintFn(constraint: leaf.getAsConstraint());
2047 assert(constraint && "prop constraint was not uniqued");
2048 return *constraint;
2049 }
2050 assert(leaf.isOperandMatcher());
2051 return staticVerifierEmitter.getTypeConstraintFn(constraint: leaf.getAsConstraint());
2052}
2053
2054static void emitRewriters(const RecordKeeper &records, raw_ostream &os) {
2055 emitSourceFileHeader(Desc: "Rewriters", OS&: os, Record: records);
2056
2057 auto patterns = records.getAllDerivedDefinitions(ClassName: "Pattern");
2058
2059 // We put the map here because it can be shared among multiple patterns.
2060 RecordOperatorMap recordOpMap;
2061
2062 // Exam all the patterns and generate static matcher for the duplicated
2063 // DagNode.
2064 StaticMatcherHelper staticMatcher(os, records, recordOpMap);
2065 for (const Record *p : patterns)
2066 staticMatcher.addPattern(record: p);
2067 staticMatcher.populateStaticConstraintFunctions(os);
2068 staticMatcher.populateStaticMatchers(os);
2069
2070 std::vector<std::string> rewriterNames;
2071 rewriterNames.reserve(n: patterns.size());
2072
2073 std::string baseRewriterName = "GeneratedConvert";
2074 int rewriterIndex = 0;
2075
2076 for (const Record *p : patterns) {
2077 std::string name;
2078 if (p->isAnonymous()) {
2079 // If no name is provided, ensure unique rewriter names simply by
2080 // appending unique suffix.
2081 name = baseRewriterName + llvm::utostr(X: rewriterIndex++);
2082 } else {
2083 name = std::string(p->getName());
2084 }
2085 LLVM_DEBUG(llvm::dbgs()
2086 << "=== start generating pattern '" << name << "' ===\n");
2087 PatternEmitter(p, &recordOpMap, os, staticMatcher).emit(rewriteName: name);
2088 LLVM_DEBUG(llvm::dbgs()
2089 << "=== done generating pattern '" << name << "' ===\n");
2090 rewriterNames.push_back(x: std::move(name));
2091 }
2092
2093 // Emit function to add the generated matchers to the pattern list.
2094 os << "[[maybe_unused]] void populateWithGenerated("
2095 "::mlir::RewritePatternSet &patterns) {\n";
2096 for (const auto &name : rewriterNames) {
2097 os << " patterns.add<" << name << ">(patterns.getContext());\n";
2098 }
2099 os << "}\n";
2100}
2101
2102static mlir::GenRegistration
2103 genRewriters("gen-rewriters", "Generate pattern rewriters",
2104 [](const RecordKeeper &records, raw_ostream &os) {
2105 emitRewriters(records, os);
2106 return false;
2107 });
2108