1//===- Pass.cpp - Pass infrastructure implementation ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements common pass infrastructure.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Pass/Pass.h"
14#include "PassDetail.h"
15#include "mlir/IR/Diagnostics.h"
16#include "mlir/IR/OpDefinition.h"
17#include "mlir/IR/Threading.h"
18#include "mlir/IR/Verifier.h"
19#include "mlir/Support/FileUtilities.h"
20#include "mlir/Support/IndentedOstream.h"
21#include "llvm/ADT/Hashing.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/Support/DebugLog.h"
25#include "llvm/Support/Mutex.h"
26#include "llvm/Support/Signals.h"
27#include "llvm/Support/Threading.h"
28#include <mutex>
29#include <optional>
30
31#define DEBUG_TYPE "pass-manager"
32
33using namespace mlir;
34using namespace mlir::detail;
35
36//===----------------------------------------------------------------------===//
37// PassExecutionAction
38//===----------------------------------------------------------------------===//
39
40PassExecutionAction::PassExecutionAction(ArrayRef<IRUnit> irUnits,
41 const Pass &pass)
42 : Base(irUnits), pass(pass) {}
43
44void PassExecutionAction::print(raw_ostream &os) const {
45 os << llvm::formatv(Fmt: "`{0}` running `{1}` on Operation `{2}`", Vals: tag,
46 Vals: pass.getName(), Vals: getOp()->getName());
47}
48
49Operation *PassExecutionAction::getOp() const {
50 ArrayRef<IRUnit> irUnits = getContextIRUnits();
51 return irUnits.empty() ? nullptr
52 : llvm::dyn_cast_if_present<Operation *>(Val: irUnits[0]);
53}
54
55MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::PassExecutionAction)
56
57//===----------------------------------------------------------------------===//
58// Pass
59//===----------------------------------------------------------------------===//
60
61/// Out of line virtual method to ensure vtables and metadata are emitted to a
62/// single .o file.
63void Pass::anchor() {}
64
65/// Attempt to initialize the options of this pass from the given string.
66LogicalResult Pass::initializeOptions(
67 StringRef options,
68 function_ref<LogicalResult(const Twine &)> errorHandler) {
69 std::string errStr;
70 llvm::raw_string_ostream os(errStr);
71 if (failed(Result: passOptions.parseFromString(options, errorStream&: os))) {
72 return errorHandler(errStr);
73 }
74 return success();
75}
76
77/// Copy the option values from 'other', which is another instance of this
78/// pass.
79void Pass::copyOptionValuesFrom(const Pass *other) {
80 passOptions.copyOptionValuesFrom(other: other->passOptions);
81}
82
83/// Prints out the pass in the textual representation of pipelines. If this is
84/// an adaptor pass, print its pass managers. When `pretty` is true, the
85/// printed pipeline is formatted for readability.
86void Pass::printAsTextualPipeline(raw_ostream &os, bool pretty) {
87 // Special case for adaptors to print its pass managers.
88 if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: this)) {
89 llvm::interleave(
90 c: adaptor->getPassManagers(),
91 each_fn: [&](OpPassManager &pm) { pm.printAsTextualPipeline(os, pretty); },
92 between_fn: [&] {
93 os << ",";
94 if (pretty)
95 os << "\n";
96 });
97 return;
98 }
99 // Otherwise, print the pass argument followed by its options. If the pass
100 // doesn't have an argument, print the name of the pass to give some indicator
101 // of what pass was run.
102 StringRef argument = getArgument();
103 if (!argument.empty())
104 os << argument;
105 else
106 os << "unknown<" << getName() << ">";
107 passOptions.print(os);
108}
109
110//===----------------------------------------------------------------------===//
111// OpPassManagerImpl
112//===----------------------------------------------------------------------===//
113
114namespace mlir {
115namespace detail {
116struct OpPassManagerImpl {
117 OpPassManagerImpl(OperationName opName, OpPassManager::Nesting nesting)
118 : name(opName.getStringRef().str()), opName(opName),
119 initializationGeneration(0), nesting(nesting) {}
120 OpPassManagerImpl(StringRef name, OpPassManager::Nesting nesting)
121 : name(name == OpPassManager::getAnyOpAnchorName() ? "" : name.str()),
122 initializationGeneration(0), nesting(nesting) {}
123 OpPassManagerImpl(OpPassManager::Nesting nesting)
124 : initializationGeneration(0), nesting(nesting) {}
125 OpPassManagerImpl(const OpPassManagerImpl &rhs)
126 : name(rhs.name), opName(rhs.opName),
127 initializationGeneration(rhs.initializationGeneration),
128 nesting(rhs.nesting) {
129 for (const std::unique_ptr<Pass> &pass : rhs.passes) {
130 std::unique_ptr<Pass> newPass = pass->clone();
131 newPass->threadingSibling = pass.get();
132 passes.push_back(x: std::move(newPass));
133 }
134 }
135
136 /// Merge the passes of this pass manager into the one provided.
137 void mergeInto(OpPassManagerImpl &rhs);
138
139 /// Nest a new operation pass manager for the given operation kind under this
140 /// pass manager.
141 OpPassManager &nest(OperationName nestedName) {
142 return nest(nested: OpPassManager(nestedName, nesting));
143 }
144 OpPassManager &nest(StringRef nestedName) {
145 return nest(nested: OpPassManager(nestedName, nesting));
146 }
147 OpPassManager &nestAny() { return nest(nested: OpPassManager(nesting)); }
148
149 /// Nest the given pass manager under this pass manager.
150 OpPassManager &nest(OpPassManager &&nested);
151
152 /// Add the given pass to this pass manager. If this pass has a concrete
153 /// operation type, it must be the same type as this pass manager.
154 void addPass(std::unique_ptr<Pass> pass);
155
156 /// Clear the list of passes in this pass manager, other options are
157 /// preserved.
158 void clear();
159
160 /// Finalize the pass list in preparation for execution. This includes
161 /// coalescing adjacent pass managers when possible, verifying scheduled
162 /// passes, etc.
163 LogicalResult finalizePassList(MLIRContext *ctx);
164
165 /// Return the operation name of this pass manager.
166 std::optional<OperationName> getOpName(MLIRContext &context) {
167 if (!name.empty() && !opName)
168 opName = OperationName(name, &context);
169 return opName;
170 }
171 std::optional<StringRef> getOpName() const {
172 return name.empty() ? std::optional<StringRef>()
173 : std::optional<StringRef>(name);
174 }
175
176 /// Return the name used to anchor this pass manager. This is either the name
177 /// of an operation, or the result of `getAnyOpAnchorName()` in the case of an
178 /// op-agnostic pass manager.
179 StringRef getOpAnchorName() const {
180 return getOpName().value_or(u: OpPassManager::getAnyOpAnchorName());
181 }
182
183 /// Indicate if the current pass manager can be scheduled on the given
184 /// operation type.
185 bool canScheduleOn(MLIRContext &context, OperationName opName);
186
187 /// The name of the operation that passes of this pass manager operate on.
188 std::string name;
189
190 /// The cached OperationName (internalized in the context) for the name of the
191 /// operation that passes of this pass manager operate on.
192 std::optional<OperationName> opName;
193
194 /// The set of passes to run as part of this pass manager.
195 std::vector<std::unique_ptr<Pass>> passes;
196
197 /// The current initialization generation of this pass manager. This is used
198 /// to indicate when a pass manager should be reinitialized.
199 unsigned initializationGeneration;
200
201 /// Control the implicit nesting of passes that mismatch the name set for this
202 /// OpPassManager.
203 OpPassManager::Nesting nesting;
204};
205} // namespace detail
206} // namespace mlir
207
208void OpPassManagerImpl::mergeInto(OpPassManagerImpl &rhs) {
209 assert(name == rhs.name && "merging unrelated pass managers");
210 for (auto &pass : passes)
211 rhs.passes.push_back(x: std::move(pass));
212 passes.clear();
213}
214
215OpPassManager &OpPassManagerImpl::nest(OpPassManager &&nested) {
216 auto *adaptor = new OpToOpPassAdaptor(std::move(nested));
217 addPass(pass: std::unique_ptr<Pass>(adaptor));
218 return adaptor->getPassManagers().front();
219}
220
221void OpPassManagerImpl::addPass(std::unique_ptr<Pass> pass) {
222 // If this pass runs on a different operation than this pass manager, then
223 // implicitly nest a pass manager for this operation if enabled.
224 std::optional<StringRef> pmOpName = getOpName();
225 std::optional<StringRef> passOpName = pass->getOpName();
226 if (pmOpName && passOpName && *pmOpName != *passOpName) {
227 if (nesting == OpPassManager::Nesting::Implicit)
228 return nest(nestedName: *passOpName).addPass(pass: std::move(pass));
229 llvm::report_fatal_error(reason: llvm::Twine("Can't add pass '") + pass->getName() +
230 "' restricted to '" + *passOpName +
231 "' on a PassManager intended to run on '" +
232 getOpAnchorName() + "', did you intend to nest?");
233 }
234
235 passes.emplace_back(args: std::move(pass));
236}
237
238void OpPassManagerImpl::clear() { passes.clear(); }
239
240LogicalResult OpPassManagerImpl::finalizePassList(MLIRContext *ctx) {
241 auto finalizeAdaptor = [ctx](OpToOpPassAdaptor *adaptor) {
242 for (auto &pm : adaptor->getPassManagers())
243 if (failed(Result: pm.getImpl().finalizePassList(ctx)))
244 return failure();
245 return success();
246 };
247
248 // Walk the pass list and merge adjacent adaptors.
249 LDBG(3) << "Merging adjacent adaptors in pass list";
250 OpToOpPassAdaptor *lastAdaptor = nullptr;
251 for (auto &pass : passes) {
252 // Check to see if this pass is an adaptor.
253 if (auto *currentAdaptor = dyn_cast<OpToOpPassAdaptor>(Val: pass.get())) {
254 // If it is the first adaptor in a possible chain, remember it and
255 // continue.
256 if (!lastAdaptor) {
257 LDBG(3) << "Found first adaptor in chain";
258 lastAdaptor = currentAdaptor;
259 continue;
260 }
261
262 // Otherwise, try to merge into the existing adaptor and delete the
263 // current one. If merging fails, just remember this as the last adaptor.
264 LDBG(3) << "Attempting to merge adaptor with "
265 << currentAdaptor->getPassManagers().size()
266 << " managers into previous adaptor";
267 if (succeeded(Result: currentAdaptor->tryMergeInto(ctx, rhs&: *lastAdaptor))) {
268 LDBG(3) << "Successfully merged adaptors, removing current one";
269 pass.reset();
270 } else {
271 LDBG(3) << "Failed to merge adaptors, keeping current as last";
272 lastAdaptor = currentAdaptor;
273 }
274 } else if (lastAdaptor) {
275 // If this pass isn't an adaptor, finalize it and forget the last adaptor.
276 LDBG(3) << "Finalizing adaptor chain before non-adaptor pass";
277 if (failed(Result: finalizeAdaptor(lastAdaptor)))
278 return failure();
279 lastAdaptor = nullptr;
280 }
281 }
282
283 // If there was an adaptor at the end of the manager, finalize it as well.
284 if (lastAdaptor && failed(Result: finalizeAdaptor(lastAdaptor)))
285 return failure();
286
287 // Now that the adaptors have been merged, erase any empty slots corresponding
288 // to the merged adaptors that were nulled-out in the loop above.
289 size_t beforeErase = passes.size();
290 llvm::erase_if(C&: passes, P: std::logical_not<std::unique_ptr<Pass>>());
291 if (beforeErase != passes.size()) {
292 LDBG(3) << "Removed " << (beforeErase - passes.size())
293 << " merged adaptor slots from pass list";
294 }
295
296 // If this is a op-agnostic pass manager, there is nothing left to do.
297 std::optional<OperationName> rawOpName = getOpName(context&: *ctx);
298 if (!rawOpName) {
299 LDBG(3)
300 << "Op-agnostic pass manager, skipping operation-specific verification";
301 return success();
302 }
303
304 // Otherwise, verify that all of the passes are valid for the current
305 // operation anchor.
306 LDBG(3) << "Verifying " << passes.size() << " passes for operation '"
307 << getOpAnchorName() << "'";
308
309 std::optional<RegisteredOperationName> opName =
310 rawOpName->getRegisteredInfo();
311 for (std::unique_ptr<Pass> &pass : passes) {
312 if (opName && !pass->canScheduleOn(opName: *opName)) {
313 return emitError(loc: UnknownLoc::get(context: ctx))
314 << "unable to schedule pass '" << pass->getName()
315 << "' on a PassManager intended to run on '" << getOpAnchorName()
316 << "'!";
317 }
318 }
319
320 LDBG(3) << "Pass list finalization completed successfully";
321 return success();
322}
323
324bool OpPassManagerImpl::canScheduleOn(MLIRContext &context,
325 OperationName opName) {
326 // If this pass manager is op-specific, we simply check if the provided
327 // operation name is the same as this one.
328 std::optional<OperationName> pmOpName = getOpName(context);
329 if (pmOpName)
330 return pmOpName == opName;
331
332 // Otherwise, this is an op-agnostic pass manager. Check that the operation
333 // can be scheduled on all passes within the manager.
334 std::optional<RegisteredOperationName> registeredInfo =
335 opName.getRegisteredInfo();
336 if (!registeredInfo ||
337 !registeredInfo->hasTrait<OpTrait::IsIsolatedFromAbove>())
338 return false;
339 return llvm::all_of(Range&: passes, P: [&](const std::unique_ptr<Pass> &pass) {
340 return pass->canScheduleOn(opName: *registeredInfo);
341 });
342}
343
344//===----------------------------------------------------------------------===//
345// OpPassManager
346//===----------------------------------------------------------------------===//
347
348OpPassManager::OpPassManager(Nesting nesting)
349 : impl(new OpPassManagerImpl(nesting)) {}
350OpPassManager::OpPassManager(StringRef name, Nesting nesting)
351 : impl(new OpPassManagerImpl(name, nesting)) {}
352OpPassManager::OpPassManager(OperationName name, Nesting nesting)
353 : impl(new OpPassManagerImpl(name, nesting)) {}
354OpPassManager::OpPassManager(OpPassManager &&rhs) { *this = std::move(rhs); }
355OpPassManager::OpPassManager(const OpPassManager &rhs) { *this = rhs; }
356OpPassManager &OpPassManager::operator=(const OpPassManager &rhs) {
357 impl = std::make_unique<OpPassManagerImpl>(args&: *rhs.impl);
358 return *this;
359}
360OpPassManager &OpPassManager::operator=(OpPassManager &&rhs) {
361 impl = std::move(rhs.impl);
362 return *this;
363}
364
365OpPassManager::~OpPassManager() = default;
366
367OpPassManager::pass_iterator OpPassManager::begin() {
368 return MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin();
369}
370OpPassManager::pass_iterator OpPassManager::end() {
371 return MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end();
372}
373
374OpPassManager::const_pass_iterator OpPassManager::begin() const {
375 return ArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin();
376}
377OpPassManager::const_pass_iterator OpPassManager::end() const {
378 return ArrayRef<std::unique_ptr<Pass>>{impl->passes}.end();
379}
380
381/// Nest a new operation pass manager for the given operation kind under this
382/// pass manager.
383OpPassManager &OpPassManager::nest(OperationName nestedName) {
384 return impl->nest(nestedName);
385}
386OpPassManager &OpPassManager::nest(StringRef nestedName) {
387 return impl->nest(nestedName);
388}
389OpPassManager &OpPassManager::nestAny() { return impl->nestAny(); }
390
391/// Add the given pass to this pass manager. If this pass has a concrete
392/// operation type, it must be the same type as this pass manager.
393void OpPassManager::addPass(std::unique_ptr<Pass> pass) {
394 impl->addPass(pass: std::move(pass));
395}
396
397void OpPassManager::clear() { impl->clear(); }
398
399/// Returns the number of passes held by this manager.
400size_t OpPassManager::size() const { return impl->passes.size(); }
401
402/// Returns the internal implementation instance.
403OpPassManagerImpl &OpPassManager::getImpl() { return *impl; }
404
405/// Return the operation name that this pass manager operates on.
406std::optional<StringRef> OpPassManager::getOpName() const {
407 return impl->getOpName();
408}
409
410/// Return the operation name that this pass manager operates on.
411std::optional<OperationName>
412OpPassManager::getOpName(MLIRContext &context) const {
413 return impl->getOpName(context);
414}
415
416StringRef OpPassManager::getOpAnchorName() const {
417 return impl->getOpAnchorName();
418}
419
420/// Prints out the passes of the pass manager as the textual representation
421/// of pipelines. When `pretty` is true, the printed pipeline is formatted for
422/// readability.
423void printAsTextualPipeline(
424 raw_indented_ostream &os, StringRef anchorName,
425 const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
426 bool pretty = false) {
427 os << anchorName << "(";
428 if (pretty) {
429 os << "\n";
430 os.indent();
431 }
432 llvm::interleave(
433 c: passes,
434 each_fn: [&](mlir::Pass &pass) { pass.printAsTextualPipeline(os, pretty); },
435 between_fn: [&]() {
436 os << ",";
437 if (pretty)
438 os << "\n";
439 });
440 if (pretty) {
441 os << "\n";
442 os.unindent();
443 }
444 os << ")";
445}
446void printAsTextualPipeline(
447 raw_ostream &os, StringRef anchorName,
448 const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
449 bool pretty) {
450 raw_indented_ostream indentedOS(os);
451 printAsTextualPipeline(os&: indentedOS, anchorName, passes, pretty);
452}
453void OpPassManager::printAsTextualPipeline(raw_ostream &os, bool pretty) const {
454 StringRef anchorName = getOpAnchorName();
455 raw_indented_ostream indentedOS(os);
456 ::printAsTextualPipeline(
457 os&: indentedOS, anchorName,
458 passes: {MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin(),
459 MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end()},
460 pretty);
461}
462
463#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
464void OpPassManager::dump() {
465 llvm::errs() << "Pass Manager with " << impl->passes.size() << " passes:\n";
466 printAsTextualPipeline(llvm::errs(), /*pretty=*/true);
467 llvm::errs() << "\n";
468}
469#endif
470
471static void registerDialectsForPipeline(const OpPassManager &pm,
472 DialectRegistry &dialects) {
473 for (const Pass &pass : pm.getPasses())
474 pass.getDependentDialects(registry&: dialects);
475}
476
477void OpPassManager::getDependentDialects(DialectRegistry &dialects) const {
478 registerDialectsForPipeline(pm: *this, dialects);
479}
480
481void OpPassManager::setNesting(Nesting nesting) { impl->nesting = nesting; }
482
483OpPassManager::Nesting OpPassManager::getNesting() { return impl->nesting; }
484
485LogicalResult OpPassManager::initialize(MLIRContext *context,
486 unsigned newInitGeneration) {
487
488 if (impl->initializationGeneration == newInitGeneration) {
489 LDBG(2) << "Pass manager already initialized "
490 << "' (generation " << newInitGeneration << ") with " << size()
491 << " passes";
492 return success();
493 }
494
495 LDBG(2) << "Initializing pass manager '" << getOpAnchorName()
496 << "' (generation " << newInitGeneration << ") with " << size()
497 << " passes";
498 impl->initializationGeneration = newInitGeneration;
499
500 for (Pass &pass : getPasses()) {
501 // If this pass isn't an adaptor, directly initialize it.
502 auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: &pass);
503 if (!adaptor) {
504 LDBG(2) << "Initializing pass '" << pass.getName() << "'";
505 if (failed(Result: pass.initialize(context))) {
506 LDBG(2) << "Failed to initialize pass '" << pass.getName() << "'";
507 return failure();
508 }
509 continue;
510 }
511
512 // Otherwise, initialize each of the adaptors pass managers.
513 LDBG(3) << "Initializing adaptor pass with "
514 << adaptor->getPassManagers().size() << " nested managers";
515 for (OpPassManager &adaptorPM : adaptor->getPassManagers())
516 if (failed(Result: adaptorPM.initialize(context, newInitGeneration))) {
517 LDBG(2) << "Failed to initialize nested pass manager";
518 return failure();
519 }
520 }
521
522 LDBG_OS([&](raw_ostream &os) {
523 os << "Pass manager initialization completed successfully: ";
524 printAsTextualPipeline(os, /*pretty=*/false);
525 });
526 return success();
527}
528
529llvm::hash_code OpPassManager::hash() {
530 llvm::hash_code hashCode{};
531 for (Pass &pass : getPasses()) {
532 // If this pass isn't an adaptor, directly hash it.
533 auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: &pass);
534 if (!adaptor) {
535 hashCode = llvm::hash_combine(args: hashCode, args: &pass);
536 continue;
537 }
538 // Otherwise, hash recursively each of the adaptors pass managers.
539 for (OpPassManager &adaptorPM : adaptor->getPassManagers())
540 llvm::hash_combine(args: hashCode, args: adaptorPM.hash());
541 }
542 return hashCode;
543}
544
545//===----------------------------------------------------------------------===//
546// OpToOpPassAdaptor
547//===----------------------------------------------------------------------===//
548
549LogicalResult OpToOpPassAdaptor::run(Pass *pass, Operation *op,
550 AnalysisManager am, bool verifyPasses,
551 unsigned parentInitGeneration) {
552 LDBG() << "Running pass '" << pass->getName() << "' on operation '"
553 << OpWithFlags(op, OpPrintingFlags().skipRegions()) << "' at "
554 << op->getLoc();
555
556 std::optional<RegisteredOperationName> opInfo = op->getRegisteredInfo();
557 if (!opInfo) {
558 return op->emitOpError()
559 << "trying to schedule a pass on an unregistered operation";
560 }
561 if (!opInfo->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
562 return op->emitOpError() << "trying to schedule a pass on an operation not "
563 "marked as 'IsolatedFromAbove'";
564 }
565 if (!pass->canScheduleOn(op)) {
566 return op->emitOpError() << "trying to schedule pass '" << pass->getName()
567 << "' on an unsupported operation";
568 }
569
570 // Initialize the pass state with a callback for the pass to dynamically
571 // execute a pipeline on the currently visited operation.
572 PassInstrumentor *pi = am.getPassInstrumentor();
573 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
574 .parentPass: pass};
575 auto dynamicPipelineCallback = [&](OpPassManager &pipeline,
576 Operation *root) -> LogicalResult {
577 if (!op->isAncestor(other: root))
578 return root->emitOpError()
579 << "Trying to schedule a dynamic pipeline on an "
580 "operation that isn't "
581 "nested under the current operation the pass is processing";
582 assert(
583 pipeline.getImpl().canScheduleOn(*op->getContext(), root->getName()));
584
585 // Before running, finalize the passes held by the pipeline.
586 if (failed(Result: pipeline.getImpl().finalizePassList(ctx: root->getContext()))) {
587 LDBG() << "Failed to finalize pass list for pipeline";
588 return failure();
589 }
590
591 // Initialize the user provided pipeline and execute the pipeline.
592 if (failed(Result: pipeline.initialize(context: root->getContext(), newInitGeneration: parentInitGeneration)))
593 return failure();
594 AnalysisManager nestedAm = root == op ? am : am.nest(op: root);
595 return OpToOpPassAdaptor::runPipeline(pm&: pipeline, op: root, am: nestedAm,
596 verifyPasses, parentInitGeneration,
597 instrumentor: pi, parentInfo: &parentInfo);
598 };
599 pass->passState.emplace(args&: op, args&: am, args&: dynamicPipelineCallback);
600
601 // Instrument before the pass has run.
602 if (pi)
603 pi->runBeforePass(pass, op);
604
605 // Pass instrumentation can use pass failure to flag unmet invariants
606 // (preconditions) of the pass. Skip running pass if in failure state.
607 bool passFailed = pass->passState->irAndPassFailed.getInt();
608 if (!passFailed) {
609 op->getContext()->executeAction<PassExecutionAction>(
610 actionFn: [&]() {
611 // Invoke the virtual runOnOperation method.
612 if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: pass))
613 adaptor->runOnOperation(verifyPasses);
614 else
615 pass->runOnOperation();
616 passFailed = pass->passState->irAndPassFailed.getInt();
617 },
618 irUnits: {op}, args&: *pass);
619 }
620
621 // Invalidate any non preserved analyses.
622 am.invalidate(pa: pass->passState->preservedAnalyses);
623
624 // When verifyPasses is specified, we run the verifier (unless the pass
625 // failed).
626 if (!passFailed && verifyPasses) {
627 bool runVerifierNow = true;
628
629 // If the pass is an adaptor pass, we don't run the verifier recursively
630 // because the nested operations should have already been verified after
631 // nested passes had run.
632 bool runVerifierRecursively = !isa<OpToOpPassAdaptor>(Val: pass);
633
634 // Reduce compile time by avoiding running the verifier if the pass didn't
635 // change the IR since the last time the verifier was run:
636 //
637 // 1) If the pass said that it preserved all analyses then it can't have
638 // permuted the IR.
639 //
640 // We run these checks in EXPENSIVE_CHECKS mode out of caution.
641#ifndef EXPENSIVE_CHECKS
642 runVerifierNow = !pass->passState->preservedAnalyses.isAll();
643#endif
644 if (runVerifierNow)
645 passFailed = failed(Result: verify(op, verifyRecursively: runVerifierRecursively));
646 }
647
648 // Instrument after the pass has run.
649 if (pi) {
650 if (passFailed) {
651 pi->runAfterPassFailed(pass, op);
652 } else {
653 pi->runAfterPass(pass, op);
654 passFailed = passFailed || pass->passState->irAndPassFailed.getInt();
655 }
656 }
657
658 // Return if the pass signaled a failure.
659 return failure(IsFailure: passFailed);
660}
661
662/// Run the given operation and analysis manager on a provided op pass manager.
663LogicalResult OpToOpPassAdaptor::runPipeline(
664 OpPassManager &pm, Operation *op, AnalysisManager am, bool verifyPasses,
665 unsigned parentInitGeneration, PassInstrumentor *instrumentor,
666 const PassInstrumentation::PipelineParentInfo *parentInfo) {
667 LDBG_OS([&](raw_ostream &os) {
668 os << "Running pipeline on operation '"
669 << OpWithFlags(op, OpPrintingFlags().skipRegions()) << "' with "
670 << pm.size() << " passes, verifyPasses=" << verifyPasses
671 << " pipeline: ";
672 pm.printAsTextualPipeline(os, /*pretty=*/false);
673 });
674 assert((!instrumentor || parentInfo) &&
675 "expected parent info if instrumentor is provided");
676 llvm::scope_exit scopeExit([&] {
677 // Clear out any computed operation analyses. These analyses won't be used
678 // any more in this pipeline, and this helps reduce the current working set
679 // of memory. If preserving these analyses becomes important in the future
680 // we can re-evaluate this.
681 am.clear();
682 });
683
684 // Run the pipeline over the provided operation.
685 if (instrumentor) {
686 instrumentor->runBeforePipeline(name: pm.getOpName(context&: *op->getContext()),
687 parentInfo: *parentInfo);
688 }
689
690 for (Pass &pass : pm.getPasses()) {
691 if (failed(Result: run(pass: &pass, op, am, verifyPasses, parentInitGeneration))) {
692 LDBG() << "Pipeline failed for pass '" << pass.getName()
693 << "' on operation '"
694 << OpWithFlags(op, OpPrintingFlags().skipRegions()) << "'";
695 return failure();
696 }
697 }
698
699 if (instrumentor) {
700 instrumentor->runAfterPipeline(name: pm.getOpName(context&: *op->getContext()),
701 parentInfo: *parentInfo);
702 }
703 return success();
704}
705
706/// Find an operation pass manager with the given anchor name, or nullptr if one
707/// does not exist.
708static OpPassManager *
709findPassManagerWithAnchor(MutableArrayRef<OpPassManager> mgrs, StringRef name) {
710 LDBG(3) << "Looking for pass manager with anchor name '" << name << "' among "
711 << mgrs.size() << " managers";
712
713 auto *it = llvm::find_if(
714 Range&: mgrs, P: [&](OpPassManager &mgr) { return mgr.getOpAnchorName() == name; });
715
716 if (it == mgrs.end()) {
717 LDBG(2) << "No pass manager found with anchor name '" << name << "'";
718 return nullptr;
719 }
720
721 LDBG(2) << "Found pass manager with anchor name '" << name << "'";
722 return &*it;
723}
724
725/// Find an operation pass manager that can operate on an operation of the given
726/// type, or nullptr if one does not exist.
727static OpPassManager *findPassManagerFor(MutableArrayRef<OpPassManager> mgrs,
728 OperationName name,
729 MLIRContext &context) {
730 LDBG(4) << "Looking for pass manager that can handle operation '" << name
731 << "' among " << mgrs.size() << " managers";
732
733 auto *it = llvm::find_if(Range&: mgrs, P: [&](OpPassManager &mgr) {
734 return mgr.getImpl().canScheduleOn(context, opName: name);
735 });
736
737 if (it == mgrs.end()) {
738 LDBG(4) << "No pass manager found that can handle operation '" << name
739 << "'";
740 return nullptr;
741 }
742
743 LDBG(4) << "Found pass manager '" << it->getOpAnchorName()
744 << "' that can handle operation '" << name << "'";
745 return &*it;
746}
747
748OpToOpPassAdaptor::OpToOpPassAdaptor(OpPassManager &&mgr) {
749 mgrs.emplace_back(Args: std::move(mgr));
750}
751
752void OpToOpPassAdaptor::getDependentDialects(DialectRegistry &dialects) const {
753 for (auto &pm : mgrs)
754 pm.getDependentDialects(dialects);
755}
756
757LogicalResult OpToOpPassAdaptor::tryMergeInto(MLIRContext *ctx,
758 OpToOpPassAdaptor &rhs) {
759 LDBG(3) << "Attempting to merge pass adaptor with " << mgrs.size()
760 << " managers into rhs with " << rhs.mgrs.size() << " managers";
761
762 // Functor used to check if a pass manager is generic, i.e. op-agnostic.
763 auto isGenericPM = [&](OpPassManager &pm) { return !pm.getOpName(); };
764
765 // Functor used to detect if the given generic pass manager will have a
766 // potential schedule conflict with the given `otherPMs`.
767 auto hasScheduleConflictWith = [&](OpPassManager &genericPM,
768 MutableArrayRef<OpPassManager> otherPMs) {
769 return llvm::any_of(Range&: otherPMs, P: [&](OpPassManager &pm) {
770 // If this is a non-generic pass manager, a conflict will arise if a
771 // non-generic pass manager's operation name can be scheduled on the
772 // generic passmanager.
773 if (std::optional<OperationName> pmOpName = pm.getOpName(context&: *ctx))
774 return genericPM.getImpl().canScheduleOn(context&: *ctx, opName: *pmOpName);
775 // Otherwise, this is a generic pass manager. We current can't determine
776 // when generic pass managers can be merged, so conservatively assume they
777 // conflict.
778 return true;
779 });
780 };
781
782 // Check that if either adaptor has a generic pass manager, that pm is
783 // compatible within any non-generic pass managers.
784 //
785 // Check the current adaptor.
786 auto *lhsGenericPMIt = llvm::find_if(Range&: mgrs, P: isGenericPM);
787 if (lhsGenericPMIt != mgrs.end()) {
788 LDBG(4) << "Found generic pass manager on LHS, checking for conflicts";
789 if (hasScheduleConflictWith(*lhsGenericPMIt, rhs.mgrs)) {
790 LDBG(4)
791 << "Merge failed: LHS generic pass manager has conflicts with RHS";
792 return failure();
793 }
794 }
795 // Check the rhs adaptor.
796 auto *rhsGenericPMIt = llvm::find_if(Range&: rhs.mgrs, P: isGenericPM);
797 if (rhsGenericPMIt != rhs.mgrs.end()) {
798 LDBG(4) << "Found generic pass manager on RHS, checking for conflicts";
799 if (hasScheduleConflictWith(*rhsGenericPMIt, mgrs)) {
800 LDBG(4)
801 << "Merge failed: RHS generic pass manager has conflicts with LHS";
802 return failure();
803 }
804 }
805
806 for (auto &pm : mgrs) {
807 // If an existing pass manager exists, then merge the given pass manager
808 // into it.
809 if (auto *existingPM =
810 findPassManagerWithAnchor(mgrs: rhs.mgrs, name: pm.getOpAnchorName())) {
811 pm.getImpl().mergeInto(rhs&: existingPM->getImpl());
812 } else {
813 // Otherwise, add the given pass manager to the list.
814 rhs.mgrs.emplace_back(Args: std::move(pm));
815 }
816 }
817 mgrs.clear();
818
819 // After coalescing, sort the pass managers within rhs by name.
820 auto compareFn = [](const OpPassManager &lhs, const OpPassManager &rhs) {
821 // Order op-specific pass managers first and op-agnostic pass managers last.
822 if (std::optional<StringRef> lhsName = lhs.getOpName()) {
823 if (std::optional<StringRef> rhsName = rhs.getOpName())
824 return *lhsName < *rhsName;
825 return true; // lhs(op-specific) < rhs(op-agnostic)
826 }
827 return false; // lhs(op-agnostic) > rhs(op-specific)
828 };
829 llvm::sort(C&: rhs.mgrs, Comp: compareFn);
830 return success();
831}
832
833/// Returns the adaptor pass name.
834std::string OpToOpPassAdaptor::getAdaptorName() {
835 std::string name = "Pipeline Collection : [";
836 llvm::raw_string_ostream os(name);
837 llvm::interleaveComma(c: getPassManagers(), os, each_fn: [&](OpPassManager &pm) {
838 os << '\'' << pm.getOpAnchorName() << '\'';
839 });
840 os << ']';
841 return name;
842}
843
844void OpToOpPassAdaptor::runOnOperation() {
845 llvm_unreachable(
846 "Unexpected call to Pass::runOnOperation() on OpToOpPassAdaptor");
847}
848
849/// Run the held pipeline over all nested operations.
850void OpToOpPassAdaptor::runOnOperation(bool verifyPasses) {
851 if (getContext().isMultithreadingEnabled())
852 runOnOperationAsyncImpl(verifyPasses);
853 else
854 runOnOperationImpl(verifyPasses);
855}
856
857/// Run this pass adaptor synchronously.
858void OpToOpPassAdaptor::runOnOperationImpl(bool verifyPasses) {
859 LDBG_OS([&](raw_ostream &os) {
860 os << "Running pass adaptor synchronously on operation '"
861 << OpWithFlags(getOperation(), OpPrintingFlags().skipRegions())
862 << "' with " << mgrs.size()
863 << " pass managers, verifyPasses=" << verifyPasses << " pipeline: ";
864 printAsTextualPipeline(os, /*pretty=*/false);
865 });
866
867 auto am = getAnalysisManager();
868 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
869 .parentPass: this};
870 auto *instrumentor = am.getPassInstrumentor();
871
872 unsigned processedOps = 0;
873 for (auto &region : getOperation()->getRegions()) {
874 for (auto &block : region) {
875 for (auto &op : block) {
876 auto *mgr = findPassManagerFor(mgrs, name: op.getName(), context&: *op.getContext());
877 if (!mgr) {
878 LDBG(2) << "Skipping operation '"
879 << OpWithFlags(&op, OpPrintingFlags().skipRegions())
880 << "': no suitable pass manager found";
881 continue;
882 }
883
884 // Run the held pipeline over the current operation.
885 LDBG(2) << "Processing operation '"
886 << OpWithFlags(&op, OpPrintingFlags().skipRegions())
887 << "' with pass manager '" << mgr->getOpAnchorName() << "'";
888
889 unsigned initGeneration = mgr->impl->initializationGeneration;
890 if (failed(Result: runPipeline(pm&: *mgr, op: &op, am: am.nest(op: &op), verifyPasses,
891 parentInitGeneration: initGeneration, instrumentor, parentInfo: &parentInfo))) {
892 LDBG(2) << "Pipeline failed for operation '"
893 << OpWithFlags(&op, OpPrintingFlags().skipRegions()) << "'";
894 signalPassFailure();
895 } else {
896 processedOps++;
897 }
898 }
899 }
900 }
901
902 LDBG() << "Completed synchronous pass adaptor run, processed " << processedOps
903 << " operations";
904}
905
906/// Utility functor that checks if the two ranges of pass managers have a size
907/// mismatch.
908static bool hasSizeMismatch(ArrayRef<OpPassManager> lhs,
909 ArrayRef<OpPassManager> rhs) {
910 return lhs.size() != rhs.size() ||
911 llvm::any_of(Range: llvm::seq<size_t>(Begin: 0, End: lhs.size()),
912 P: [&](size_t i) { return lhs[i].size() != rhs[i].size(); });
913}
914
915/// Run this pass adaptor synchronously.
916void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
917 LDBG_OS([&](raw_ostream &os) {
918 os << "Running pass adaptor asynchronously on operation '"
919 << OpWithFlags(getOperation(), OpPrintingFlags().skipRegions())
920 << "' with " << mgrs.size()
921 << " pass managers, verifyPasses=" << verifyPasses << " pipeline: ";
922 printAsTextualPipeline(os, /*pretty=*/false);
923 });
924
925 AnalysisManager am = getAnalysisManager();
926 MLIRContext *context = &getContext();
927
928 // Create the async executors if they haven't been created, or if the main
929 // pipeline has changed.
930 if (asyncExecutors.empty() || hasSizeMismatch(lhs: asyncExecutors.front(), rhs: mgrs)) {
931 LDBG(2) << "Creating " << context->getThreadPool().getMaxConcurrency()
932 << " async executors";
933 asyncExecutors.assign(NumElts: context->getThreadPool().getMaxConcurrency(), Elt: mgrs);
934 }
935
936 // This struct represents the information for a single operation to be
937 // scheduled on a pass manager.
938 struct OpPMInfo {
939 OpPMInfo(unsigned passManagerIdx, Operation *op, AnalysisManager am)
940 : passManagerIdx(passManagerIdx), op(op), am(am) {}
941
942 /// The index of the pass manager to schedule the operation on.
943 unsigned passManagerIdx;
944 /// The operation to schedule.
945 Operation *op;
946 /// The analysis manager for the operation.
947 AnalysisManager am;
948 };
949
950 // Run a prepass over the operation to collect the nested operations to
951 // execute over. This ensures that an analysis manager exists for each
952 // operation, as well as providing a queue of operations to execute over.
953 std::vector<OpPMInfo> opInfos;
954 DenseMap<OperationName, std::optional<unsigned>> knownOpPMIdx;
955
956 LDBG(2) << "Collecting operations for async execution";
957 for (auto &region : getOperation()->getRegions()) {
958 for (Operation &op : region.getOps()) {
959 // Get the pass manager index for this operation type.
960 auto pmIdxIt = knownOpPMIdx.try_emplace(Key: op.getName(), Args: std::nullopt);
961 if (pmIdxIt.second) {
962 if (auto *mgr = findPassManagerFor(mgrs, name: op.getName(), context&: *context)) {
963 pmIdxIt.first->second = std::distance(first: mgrs.begin(), last: mgr);
964 LDBG(2) << "Operation '"
965 << OpWithFlags(&op, OpPrintingFlags().skipRegions())
966 << "' will use pass manager '" << mgr->getOpAnchorName()
967 << "'";
968 }
969 }
970
971 // If this operation can be scheduled, add it to the list.
972 if (pmIdxIt.first->second) {
973 opInfos.emplace_back(args&: *pmIdxIt.first->second, args: &op, args: am.nest(op: &op));
974 } else {
975 LDBG(2) << "Operation '"
976 << OpWithFlags(&op, OpPrintingFlags().skipRegions())
977 << "' skipped: no suitable pass manager";
978 }
979 }
980 }
981
982 LDBG(2) << "Collected " << opInfos.size()
983 << " operations for async execution";
984
985 // Get the current thread for this adaptor.
986 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
987 .parentPass: this};
988 auto *instrumentor = am.getPassInstrumentor();
989
990 // Each parallel task owns an executor for its entire lifetime. A worker
991 // index belongs to the task, rather than the thread: nested waits may run
992 // other tasks on the same thread while this task's executor is still in use.
993 std::atomic<bool> hasFailure = false;
994 (void)failableParallelForEachWithWorkerId(
995 context, begin: opInfos.begin(), end: opInfos.end(),
996 func: [&](OpPMInfo &opInfo, unsigned pmIndex) {
997 OpPassManager &pm = asyncExecutors[pmIndex][opInfo.passManagerIdx];
998 LogicalResult pipelineResult = runPipeline(
999 pm, op: opInfo.op, am: opInfo.am, verifyPasses,
1000 parentInitGeneration: pm.impl->initializationGeneration, instrumentor, parentInfo: &parentInfo);
1001 if (failed(Result: pipelineResult))
1002 hasFailure.store(i: true);
1003 // Continue processing independent operations after a pipeline failure.
1004 return success();
1005 });
1006
1007 // Signal a failure if any of the executors failed.
1008 if (hasFailure)
1009 signalPassFailure();
1010}
1011
1012//===----------------------------------------------------------------------===//
1013// PassManager
1014//===----------------------------------------------------------------------===//
1015
1016PassManager::PassManager(MLIRContext *ctx, StringRef operationName,
1017 Nesting nesting)
1018 : OpPassManager(operationName, nesting), context(ctx), passTiming(false),
1019 verifyPasses(true) {}
1020
1021PassManager::PassManager(OperationName operationName, Nesting nesting)
1022 : OpPassManager(operationName, nesting),
1023 context(operationName.getContext()), passTiming(false),
1024 verifyPasses(true) {}
1025
1026PassManager::~PassManager() = default;
1027
1028void PassManager::enableVerifier(bool enabled) { verifyPasses = enabled; }
1029
1030/// Run the passes within this manager on the provided operation.
1031LogicalResult PassManager::run(Operation *op) {
1032 LDBG_OS([&](raw_ostream &os) {
1033 os << "Starting PassManager run on operation '"
1034 << OpWithFlags(op, OpPrintingFlags().skipRegions()) << "' with "
1035 << size() << " passes, verifyPasses=" << verifyPasses << " pipeline: ";
1036 printAsTextualPipeline(os, /*pretty=*/false);
1037 });
1038
1039 MLIRContext *context = getContext();
1040 std::optional<OperationName> anchorOp = getOpName(context&: *context);
1041 if (anchorOp && anchorOp != op->getName()) {
1042 return emitError(loc: op->getLoc())
1043 << "can't run '" << getOpAnchorName() << "' pass manager on '"
1044 << op->getName() << "' op";
1045 }
1046
1047 // Register all dialects for the current pipeline.
1048 LDBG(2) << "Registering dependent dialects for pipeline";
1049 DialectRegistry dependentDialects;
1050 getDependentDialects(dialects&: dependentDialects);
1051 context->appendDialectRegistry(registry: dependentDialects);
1052 for (StringRef name : dependentDialects.getRegisteredDialectNames()) {
1053 LDBG(2) << "Loading dialect: " << name;
1054 Dialect *loaded = context->getOrLoadDialect(name);
1055 assert(loaded && "allocator-backed registration must resolve");
1056 (void)loaded;
1057 }
1058 if (failed(Result: dependentDialects.preloadSelectDialects(
1059 ctx: context, emitError: [&]() { return emitError(loc: op->getLoc()); })))
1060 return failure();
1061
1062 // Before running, make sure to finalize the pipeline pass list.
1063 if (failed(Result: getImpl().finalizePassList(ctx: context))) {
1064 LDBG(2) << "Pass list finalization failed";
1065 return failure();
1066 }
1067
1068 // Notify the context that we start running a pipeline for bookkeeping.
1069 context->enterMultiThreadedExecution();
1070
1071 // Initialize all of the passes within the pass manager with a new generation.
1072 llvm::hash_code newInitKey = context->getRegistryHash();
1073 llvm::hash_code pipelineKey = hash();
1074 if (newInitKey != initializationKey ||
1075 pipelineKey != pipelineInitializationKey) {
1076 LDBG(2) << "Initializing passes with new generation: "
1077 << (impl->initializationGeneration + 1);
1078 if (failed(Result: initialize(context, newInitGeneration: impl->initializationGeneration + 1))) {
1079 LDBG(2) << "Pass initialization failed";
1080 return failure();
1081 }
1082 initializationKey = newInitKey;
1083 pipelineInitializationKey = pipelineKey;
1084 } else {
1085 LDBG(2) << "Using existing pass initialization (generation: "
1086 << impl->initializationGeneration << ")";
1087 }
1088
1089 // Construct a top level analysis manager for the pipeline.
1090 LDBG(2) << "Constructing analysis manager for pipeline execution";
1091 ModuleAnalysisManager am(op, instrumentor.get());
1092
1093 // If reproducer generation is enabled, run the pass manager with crash
1094 // handling enabled.
1095 LDBG(2) << "Executing pipeline with "
1096 << (crashReproGenerator ? "crash recovery" : "normal execution");
1097 LogicalResult result =
1098 crashReproGenerator ? runWithCrashRecovery(op, am) : runPasses(op, am);
1099
1100 // Notify the context that the run is done.
1101 context->exitMultiThreadedExecution();
1102
1103 // Dump all of the pass statistics if necessary.
1104 if (passStatisticsMode) {
1105 LDBG(2) << "Dumping pass statistics";
1106 dumpStatistics();
1107 }
1108
1109 LDBG(2) << "PassManager run completed with result: "
1110 << (succeeded(Result: result) ? "success" : "failure");
1111 return result;
1112}
1113
1114/// Add the provided instrumentation to the pass manager.
1115void PassManager::addInstrumentation(std::unique_ptr<PassInstrumentation> pi) {
1116 if (!instrumentor)
1117 instrumentor = std::make_unique<PassInstrumentor>();
1118
1119 instrumentor->addInstrumentation(pi: std::move(pi));
1120}
1121
1122LogicalResult PassManager::runPasses(Operation *op, AnalysisManager am) {
1123 LDBG(2) << "Executing passes using OpToOpPassAdaptor pipeline";
1124 return OpToOpPassAdaptor::runPipeline(pm&: *this, op, am, verifyPasses,
1125 parentInitGeneration: impl->initializationGeneration);
1126}
1127
1128//===----------------------------------------------------------------------===//
1129// AnalysisManager
1130//===----------------------------------------------------------------------===//
1131
1132/// Get an analysis manager for the given operation, which must be a proper
1133/// descendant of the current operation represented by this analysis manager.
1134AnalysisManager AnalysisManager::nest(Operation *op) {
1135 Operation *currentOp = impl->getOperation();
1136 assert(currentOp->isProperAncestor(op) &&
1137 "expected valid descendant operation");
1138
1139 // Check for the base case where the provided operation is immediately nested.
1140 if (currentOp == op->getParentOp())
1141 return nestImmediate(op);
1142
1143 // Otherwise, we need to collect all ancestors up to the current operation.
1144 SmallVector<Operation *, 4> opAncestors;
1145 do {
1146 opAncestors.push_back(Elt: op);
1147 op = op->getParentOp();
1148 } while (op != currentOp);
1149
1150 AnalysisManager result = *this;
1151 for (Operation *op : llvm::reverse(C&: opAncestors))
1152 result = result.nestImmediate(op);
1153 return result;
1154}
1155
1156/// Get an analysis manager for the given immediately nested child operation.
1157AnalysisManager AnalysisManager::nestImmediate(Operation *op) {
1158 assert(impl->getOperation() == op->getParentOp() &&
1159 "expected immediate child operation");
1160
1161 auto [it, inserted] = impl->childAnalyses.try_emplace(Key: op);
1162 if (inserted)
1163 it->second = std::make_unique<NestedAnalysisMap>(args&: op, args&: impl);
1164 return {it->second.get()};
1165}
1166
1167/// Invalidate any non preserved analyses.
1168void detail::NestedAnalysisMap::invalidate(
1169 const detail::PreservedAnalyses &pa) {
1170 // If all analyses were preserved, then there is nothing to do here.
1171 if (pa.isAll())
1172 return;
1173
1174 // Invalidate the analyses for the current operation directly.
1175 analyses.invalidate(pa);
1176
1177 // If no analyses were preserved, then just simply clear out the child
1178 // analysis results.
1179 if (pa.isNone()) {
1180 childAnalyses.clear();
1181 return;
1182 }
1183
1184 // Otherwise, invalidate each child analysis map.
1185 SmallVector<NestedAnalysisMap *, 8> mapsToInvalidate(1, this);
1186 while (!mapsToInvalidate.empty()) {
1187 auto *map = mapsToInvalidate.pop_back_val();
1188 for (auto &analysisPair : map->childAnalyses) {
1189 analysisPair.second->invalidate(pa);
1190 if (!analysisPair.second->childAnalyses.empty())
1191 mapsToInvalidate.push_back(Elt: analysisPair.second.get());
1192 }
1193 }
1194}
1195
1196//===----------------------------------------------------------------------===//
1197// PassInstrumentation
1198//===----------------------------------------------------------------------===//
1199
1200PassInstrumentation::~PassInstrumentation() = default;
1201
1202void PassInstrumentation::runBeforePipeline(
1203 std::optional<OperationName> name, const PipelineParentInfo &parentInfo) {}
1204
1205void PassInstrumentation::runAfterPipeline(
1206 std::optional<OperationName> name, const PipelineParentInfo &parentInfo) {}
1207
1208void PassInstrumentation::signalPassFailure(Pass *pass) {
1209 pass->signalPassFailure();
1210}
1211
1212//===----------------------------------------------------------------------===//
1213// PassInstrumentor
1214//===----------------------------------------------------------------------===//
1215
1216namespace mlir {
1217namespace detail {
1218struct PassInstrumentorImpl {
1219 /// Mutex to keep instrumentation access thread-safe.
1220 llvm::sys::SmartMutex<true> mutex;
1221
1222 /// Dispatch may run concurrently only if every instrumentation opts in.
1223 /// Registration finishes before the pipeline starts running.
1224 bool threadSafe = true;
1225
1226 auto lockInstrumentations() {
1227 std::unique_lock<llvm::sys::SmartMutex<true>> lock(mutex, std::defer_lock);
1228 if (!threadSafe)
1229 lock.lock();
1230 return lock;
1231 }
1232
1233 /// Set of registered instrumentations.
1234 std::vector<std::unique_ptr<PassInstrumentation>> instrumentations;
1235};
1236} // namespace detail
1237} // namespace mlir
1238
1239PassInstrumentor::PassInstrumentor() : impl(new PassInstrumentorImpl()) {}
1240PassInstrumentor::~PassInstrumentor() = default;
1241
1242/// See PassInstrumentation::runBeforePipeline for details.
1243void PassInstrumentor::runBeforePipeline(
1244 std::optional<OperationName> name,
1245 const PassInstrumentation::PipelineParentInfo &parentInfo) {
1246 auto instrumentationLock = impl->lockInstrumentations();
1247 for (auto &instr : impl->instrumentations)
1248 instr->runBeforePipeline(name, parentInfo);
1249}
1250
1251/// See PassInstrumentation::runAfterPipeline for details.
1252void PassInstrumentor::runAfterPipeline(
1253 std::optional<OperationName> name,
1254 const PassInstrumentation::PipelineParentInfo &parentInfo) {
1255 auto instrumentationLock = impl->lockInstrumentations();
1256 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1257 instr->runAfterPipeline(name, parentInfo);
1258}
1259
1260/// See PassInstrumentation::runBeforePass for details.
1261void PassInstrumentor::runBeforePass(Pass *pass, Operation *op) {
1262 auto instrumentationLock = impl->lockInstrumentations();
1263 for (auto &instr : impl->instrumentations)
1264 instr->runBeforePass(pass, op);
1265}
1266
1267/// See PassInstrumentation::runAfterPass for details.
1268void PassInstrumentor::runAfterPass(Pass *pass, Operation *op) {
1269 auto instrumentationLock = impl->lockInstrumentations();
1270 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1271 instr->runAfterPass(pass, op);
1272}
1273
1274/// See PassInstrumentation::runAfterPassFailed for details.
1275void PassInstrumentor::runAfterPassFailed(Pass *pass, Operation *op) {
1276 auto instrumentationLock = impl->lockInstrumentations();
1277 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1278 instr->runAfterPassFailed(pass, op);
1279}
1280
1281/// See PassInstrumentation::runBeforeAnalysis for details.
1282void PassInstrumentor::runBeforeAnalysis(StringRef name, TypeID id,
1283 Operation *op) {
1284 auto instrumentationLock = impl->lockInstrumentations();
1285 for (auto &instr : impl->instrumentations)
1286 instr->runBeforeAnalysis(name, id, op);
1287}
1288
1289/// See PassInstrumentation::runAfterAnalysis for details.
1290void PassInstrumentor::runAfterAnalysis(StringRef name, TypeID id,
1291 Operation *op) {
1292 auto instrumentationLock = impl->lockInstrumentations();
1293 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1294 instr->runAfterAnalysis(name, id, op);
1295}
1296
1297/// Add the given instrumentation to the collection.
1298void PassInstrumentor::addInstrumentation(
1299 std::unique_ptr<PassInstrumentation> pi) {
1300 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1301 impl->threadSafe &= pi->isThreadSafe();
1302 impl->instrumentations.emplace_back(args: std::move(pi));
1303}
1304