1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
15#include "llvm/Support/FormatVariadic.h"
16#include "llvm/Support/MSVCErrorWorkarounds.h"
17
18#include <condition_variable>
19#include <future>
20#include <optional>
21
22#define DEBUG_TYPE "orc"
23
24namespace llvm {
25namespace orc {
26
27char ResourceTrackerDefunct::ID = 0;
28char FailedToMaterialize::ID = 0;
29char SymbolsNotFound::ID = 0;
30char SymbolsCouldNotBeRemoved::ID = 0;
31char MissingSymbolDefinitions::ID = 0;
32char UnexpectedSymbolDefinitions::ID = 0;
33char UnsatisfiedSymbolDependencies::ID = 0;
34char MaterializationTask::ID = 0;
35char LookupTask::ID = 0;
36
37RegisterDependenciesFunction NoDependenciesToRegister =
38 RegisterDependenciesFunction();
39
40void MaterializationUnit::anchor() {}
41
42ResourceTracker::ResourceTracker(JITDylibSP JD) {
43 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
44 "JITDylib must be two byte aligned");
45 JD->Retain();
46 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
47}
48
49ResourceTracker::~ResourceTracker() {
50 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
51 getJITDylib().Release();
52}
53
54Error ResourceTracker::remove() {
55 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
56}
57
58void ResourceTracker::transferTo(ResourceTracker &DstRT) {
59 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
60}
61
62void ResourceTracker::makeDefunct() {
63 uintptr_t Val = JDAndFlag.load();
64 Val |= 0x1U;
65 JDAndFlag.store(i: Val);
66}
67
68ResourceManager::~ResourceManager() = default;
69
70ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
71 : RT(std::move(RT)) {}
72
73std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
74 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
75}
76
77void ResourceTrackerDefunct::log(raw_ostream &OS) const {
78 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
79}
80
81FailedToMaterialize::FailedToMaterialize(
82 std::shared_ptr<SymbolStringPool> SSP,
83 std::shared_ptr<SymbolDependenceMap> Symbols)
84 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
85 assert(this->SSP && "String pool cannot be null");
86 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
87
88 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
89 // don't have to manually retain/release.
90 for (auto &[JD, Syms] : *this->Symbols)
91 JD->Retain();
92}
93
94FailedToMaterialize::~FailedToMaterialize() {
95 for (auto &[JD, Syms] : *Symbols)
96 JD->Release();
97}
98
99std::error_code FailedToMaterialize::convertToErrorCode() const {
100 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
101}
102
103void FailedToMaterialize::log(raw_ostream &OS) const {
104 OS << "Failed to materialize symbols: " << *Symbols;
105}
106
107UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
108 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
109 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
110 std::string Explanation)
111 : SSP(std::move(SSP)), JD(std::move(JD)),
112 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
113 Explanation(std::move(Explanation)) {}
114
115std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
116 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
117}
118
119void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
120 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
121 << ", due to unsatisfied dependencies " << BadDeps;
122 if (!Explanation.empty())
123 OS << " (" << Explanation << ")";
124}
125
126SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
127 SymbolNameSet Symbols)
128 : SSP(std::move(SSP)) {
129 for (auto &Sym : Symbols)
130 this->Symbols.push_back(x: Sym);
131 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
132}
133
134SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
135 SymbolNameVector Symbols)
136 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
137 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
138}
139
140std::error_code SymbolsNotFound::convertToErrorCode() const {
141 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
142}
143
144void SymbolsNotFound::log(raw_ostream &OS) const {
145 OS << "Symbols not found: " << Symbols;
146}
147
148SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
149 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
150 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
151 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
152}
153
154std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
155 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
156}
157
158void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
159 OS << "Symbols could not be removed: " << Symbols;
160}
161
162std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
163 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
164}
165
166void MissingSymbolDefinitions::log(raw_ostream &OS) const {
167 OS << "Missing definitions in module " << ModuleName
168 << ": " << Symbols;
169}
170
171std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
172 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
173}
174
175void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
176 OS << "Unexpected definitions in module " << ModuleName
177 << ": " << Symbols;
178}
179
180AsynchronousSymbolQuery::AsynchronousSymbolQuery(
181 const SymbolLookupSet &Symbols, SymbolState RequiredState,
182 SymbolsResolvedCallback NotifyComplete)
183 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
184 assert(RequiredState >= SymbolState::Resolved &&
185 "Cannot query for a symbols that have not reached the resolve state "
186 "yet");
187
188 OutstandingSymbolsCount = Symbols.size();
189
190 for (auto &[Name, Flags] : Symbols)
191 ResolvedSymbols[Name] = ExecutorSymbolDef();
192}
193
194void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
195 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
196 auto I = ResolvedSymbols.find(Val: Name);
197 assert(I != ResolvedSymbols.end() &&
198 "Resolving symbol outside the requested set");
199 assert(I->second == ExecutorSymbolDef() &&
200 "Redundantly resolving symbol Name");
201
202 // If this is a materialization-side-effects-only symbol then drop it,
203 // otherwise update its map entry with its resolved address.
204 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
205 ResolvedSymbols.erase(I);
206 else
207 I->second = std::move(Sym);
208 --OutstandingSymbolsCount;
209}
210
211void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
212 assert(OutstandingSymbolsCount == 0 &&
213 "Symbols remain, handleComplete called prematurely");
214
215 class RunQueryCompleteTask : public Task {
216 public:
217 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
218 SymbolsResolvedCallback NotifyComplete)
219 : ResolvedSymbols(std::move(ResolvedSymbols)),
220 NotifyComplete(std::move(NotifyComplete)) {}
221 void printDescription(raw_ostream &OS) override {
222 OS << "Execute query complete callback for " << ResolvedSymbols;
223 }
224 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
225
226 private:
227 SymbolMap ResolvedSymbols;
228 SymbolsResolvedCallback NotifyComplete;
229 };
230
231 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
232 args: std::move(NotifyComplete));
233 NotifyComplete = SymbolsResolvedCallback();
234 ES.dispatchTask(T: std::move(T));
235}
236
237void AsynchronousSymbolQuery::handleFailed(Error Err) {
238 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
239 OutstandingSymbolsCount == 0 &&
240 "Query should already have been abandoned");
241 NotifyComplete(std::move(Err));
242 NotifyComplete = SymbolsResolvedCallback();
243}
244
245void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
246 SymbolStringPtr Name) {
247 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
248 (void)Added;
249 assert(Added && "Duplicate dependence notification?");
250}
251
252void AsynchronousSymbolQuery::removeQueryDependence(
253 JITDylib &JD, const SymbolStringPtr &Name) {
254 auto QRI = QueryRegistrations.find(Val: &JD);
255 assert(QRI != QueryRegistrations.end() &&
256 "No dependencies registered for JD");
257 assert(QRI->second.count(Name) && "No dependency on Name in JD");
258 QRI->second.erase(V: Name);
259 if (QRI->second.empty())
260 QueryRegistrations.erase(I: QRI);
261}
262
263void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
264 auto I = ResolvedSymbols.find(Val: Name);
265 assert(I != ResolvedSymbols.end() &&
266 "Redundant removal of weakly-referenced symbol");
267 ResolvedSymbols.erase(I);
268 --OutstandingSymbolsCount;
269}
270
271void AsynchronousSymbolQuery::detach() {
272 ResolvedSymbols.clear();
273 OutstandingSymbolsCount = 0;
274 for (auto &[JD, Syms] : QueryRegistrations)
275 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
276 QueryRegistrations.clear();
277}
278
279AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit(
280 SymbolMap Symbols)
281 : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
282
283StringRef AbsoluteSymbolsMaterializationUnit::getName() const {
284 return "<Absolute Symbols>";
285}
286
287void AbsoluteSymbolsMaterializationUnit::materialize(
288 std::unique_ptr<MaterializationResponsibility> R) {
289 // Even though these are just absolute symbols we need to check for failure
290 // to resolve/emit: the tracker for these symbols may have been removed while
291 // the materialization was in flight (e.g. due to a failure in some action
292 // triggered by the queries attached to the resolution/emission of these
293 // symbols).
294 if (auto Err = R->notifyResolved(Symbols)) {
295 R->getExecutionSession().reportError(Err: std::move(Err));
296 R->failMaterialization();
297 return;
298 }
299 if (auto Err = R->notifyEmitted(EmittedDeps: {})) {
300 R->getExecutionSession().reportError(Err: std::move(Err));
301 R->failMaterialization();
302 return;
303 }
304}
305
306void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
307 const SymbolStringPtr &Name) {
308 assert(Symbols.count(Name) && "Symbol is not part of this MU");
309 Symbols.erase(Val: Name);
310}
311
312MaterializationUnit::Interface
313AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
314 SymbolFlagsMap Flags;
315 for (const auto &[Name, Def] : Symbols)
316 Flags[Name] = Def.getFlags();
317 return MaterializationUnit::Interface(std::move(Flags), nullptr);
318}
319
320ReExportsMaterializationUnit::ReExportsMaterializationUnit(
321 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
322 SymbolAliasMap Aliases)
323 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
324 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
325
326StringRef ReExportsMaterializationUnit::getName() const {
327 return "<Reexports>";
328}
329
330void ReExportsMaterializationUnit::materialize(
331 std::unique_ptr<MaterializationResponsibility> R) {
332
333 auto &ES = R->getTargetJITDylib().getExecutionSession();
334 JITDylib &TgtJD = R->getTargetJITDylib();
335 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
336
337 // Find the set of requested aliases and aliasees. Return any unrequested
338 // aliases back to the JITDylib so as to not prematurely materialize any
339 // aliasees.
340 auto RequestedSymbols = R->getRequestedSymbols();
341 SymbolAliasMap RequestedAliases;
342
343 for (auto &Name : RequestedSymbols) {
344 auto I = Aliases.find(Val: Name);
345 assert(I != Aliases.end() && "Symbol not found in aliases map?");
346 RequestedAliases[Name] = std::move(I->second);
347 Aliases.erase(I);
348 }
349
350 LLVM_DEBUG({
351 ES.runSessionLocked([&]() {
352 dbgs() << "materializing reexports: target = " << TgtJD.getName()
353 << ", source = " << SrcJD.getName() << " " << RequestedAliases
354 << "\n";
355 });
356 });
357
358 if (!Aliases.empty()) {
359 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
360 SourceJDLookupFlags))
361 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
362
363 if (Err) {
364 // FIXME: Should this be reported / treated as failure to materialize?
365 // Or should this be treated as a sanctioned bailing-out?
366 ES.reportError(Err: std::move(Err));
367 R->failMaterialization();
368 return;
369 }
370 }
371
372 // The OnResolveInfo struct will hold the aliases and responsibility for each
373 // query in the list.
374 struct OnResolveInfo {
375 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
376 SymbolAliasMap Aliases)
377 : R(std::move(R)), Aliases(std::move(Aliases)) {}
378
379 std::unique_ptr<MaterializationResponsibility> R;
380 SymbolAliasMap Aliases;
381 std::vector<SymbolDependenceGroup> SDGs;
382 };
383
384 // Build a list of queries to issue. In each round we build a query for the
385 // largest set of aliases that we can resolve without encountering a chain of
386 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
387 // query would be waiting on a symbol that it itself had to resolve. Creating
388 // a new query for each link in such a chain eliminates the possibility of
389 // deadlock. In practice chains are likely to be rare, and this algorithm will
390 // usually result in a single query to issue.
391
392 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
393 QueryInfos;
394 while (!RequestedAliases.empty()) {
395 SymbolNameSet ResponsibilitySymbols;
396 SymbolLookupSet QuerySymbols;
397 SymbolAliasMap QueryAliases;
398
399 // Collect as many aliases as we can without including a chain.
400 for (auto &KV : RequestedAliases) {
401 // Chain detected. Skip this symbol for this round.
402 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: KV.second.Aliasee) ||
403 RequestedAliases.count(Val: KV.second.Aliasee)))
404 continue;
405
406 ResponsibilitySymbols.insert(V: KV.first);
407 QuerySymbols.add(Name: KV.second.Aliasee,
408 Flags: KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
409 ? SymbolLookupFlags::WeaklyReferencedSymbol
410 : SymbolLookupFlags::RequiredSymbol);
411 QueryAliases[KV.first] = std::move(KV.second);
412 }
413
414 // Remove the aliases collected this round from the RequestedAliases map.
415 for (auto &KV : QueryAliases)
416 RequestedAliases.erase(Val: KV.first);
417
418 assert(!QuerySymbols.empty() && "Alias cycle detected!");
419
420 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
421 if (!NewR) {
422 ES.reportError(Err: NewR.takeError());
423 R->failMaterialization();
424 return;
425 }
426
427 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
428 args: std::move(QueryAliases));
429 QueryInfos.push_back(
430 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
431 }
432
433 // Issue the queries.
434 while (!QueryInfos.empty()) {
435 auto QuerySymbols = std::move(QueryInfos.back().first);
436 auto QueryInfo = std::move(QueryInfos.back().second);
437
438 QueryInfos.pop_back();
439
440 auto RegisterDependencies = [QueryInfo,
441 &SrcJD](const SymbolDependenceMap &Deps) {
442 // If there were no materializing symbols, just bail out.
443 if (Deps.empty())
444 return;
445
446 // Otherwise the only deps should be on SrcJD.
447 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
448 "Unexpected dependencies for reexports");
449
450 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
451
452 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
453 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
454 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
455 };
456
457 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
458 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
459 if (Result) {
460 SymbolMap ResolutionMap;
461 for (auto &KV : QueryInfo->Aliases) {
462 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
463 Result->count(KV.second.Aliasee)) &&
464 "Result map missing entry?");
465 // Don't try to resolve materialization-side-effects-only symbols.
466 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
467 continue;
468
469 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
470 KV.second.AliasFlags};
471 }
472 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
473 ES.reportError(Err: std::move(Err));
474 QueryInfo->R->failMaterialization();
475 return;
476 }
477 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
478 ES.reportError(Err: std::move(Err));
479 QueryInfo->R->failMaterialization();
480 return;
481 }
482 } else {
483 ES.reportError(Err: Result.takeError());
484 QueryInfo->R->failMaterialization();
485 }
486 };
487
488 ES.lookup(K: LookupKind::Static,
489 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
490 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
491 RegisterDependencies: std::move(RegisterDependencies));
492 }
493}
494
495void ReExportsMaterializationUnit::discard(const JITDylib &JD,
496 const SymbolStringPtr &Name) {
497 assert(Aliases.count(Name) &&
498 "Symbol not covered by this MaterializationUnit");
499 Aliases.erase(Val: Name);
500}
501
502MaterializationUnit::Interface
503ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
504 SymbolFlagsMap SymbolFlags;
505 for (auto &KV : Aliases)
506 SymbolFlags[KV.first] = KV.second.AliasFlags;
507
508 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
509}
510
511Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
512 SymbolNameSet Symbols) {
513 SymbolLookupSet LookupSet(Symbols);
514 auto Flags = SourceJD.getExecutionSession().lookupFlags(
515 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
516 Symbols: SymbolLookupSet(std::move(Symbols)));
517
518 if (!Flags)
519 return Flags.takeError();
520
521 SymbolAliasMap Result;
522 for (auto &Name : Symbols) {
523 assert(Flags->count(Name) && "Missing entry in flags map");
524 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
525 }
526
527 return Result;
528}
529
530class InProgressLookupState {
531public:
532 // FIXME: Reduce the number of SymbolStringPtrs here. See
533 // https://github.com/llvm/llvm-project/issues/55576.
534
535 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
536 SymbolLookupSet LookupSet, SymbolState RequiredState)
537 : K(K), SearchOrder(std::move(SearchOrder)),
538 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
539 DefGeneratorCandidates = this->LookupSet;
540 }
541 virtual ~InProgressLookupState() = default;
542 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
543 virtual void fail(Error Err) = 0;
544
545 LookupKind K;
546 JITDylibSearchOrder SearchOrder;
547 SymbolLookupSet LookupSet;
548 SymbolState RequiredState;
549
550 size_t CurSearchOrderIndex = 0;
551 bool NewJITDylib = true;
552 SymbolLookupSet DefGeneratorCandidates;
553 SymbolLookupSet DefGeneratorNonCandidates;
554
555 enum {
556 NotInGenerator, // Not currently using a generator.
557 ResumedForGenerator, // Resumed after being auto-suspended before generator.
558 InGenerator // Currently using generator.
559 } GenState = NotInGenerator;
560 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
561};
562
563class InProgressLookupFlagsState : public InProgressLookupState {
564public:
565 InProgressLookupFlagsState(
566 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
567 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
568 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
569 SymbolState::NeverSearched),
570 OnComplete(std::move(OnComplete)) {}
571
572 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
573 auto &ES = SearchOrder.front().first->getExecutionSession();
574 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
575 }
576
577 void fail(Error Err) override { OnComplete(std::move(Err)); }
578
579private:
580 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
581};
582
583class InProgressFullLookupState : public InProgressLookupState {
584public:
585 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
586 SymbolLookupSet LookupSet,
587 SymbolState RequiredState,
588 std::shared_ptr<AsynchronousSymbolQuery> Q,
589 RegisterDependenciesFunction RegisterDependencies)
590 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
591 RequiredState),
592 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
593 }
594
595 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
596 auto &ES = SearchOrder.front().first->getExecutionSession();
597 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
598 RegisterDependencies: std::move(RegisterDependencies));
599 }
600
601 void fail(Error Err) override {
602 Q->detach();
603 Q->handleFailed(Err: std::move(Err));
604 }
605
606private:
607 std::shared_ptr<AsynchronousSymbolQuery> Q;
608 RegisterDependenciesFunction RegisterDependencies;
609};
610
611ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
612 JITDylibLookupFlags SourceJDLookupFlags,
613 SymbolPredicate Allow)
614 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
615 Allow(std::move(Allow)) {}
616
617Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
618 JITDylib &JD,
619 JITDylibLookupFlags JDLookupFlags,
620 const SymbolLookupSet &LookupSet) {
621 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
622
623 // Use lookupFlags to find the subset of symbols that match our lookup.
624 auto Flags = JD.getExecutionSession().lookupFlags(
625 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
626 if (!Flags)
627 return Flags.takeError();
628
629 // Create an alias map.
630 orc::SymbolAliasMap AliasMap;
631 for (auto &KV : *Flags)
632 if (!Allow || Allow(KV.first))
633 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
634
635 if (AliasMap.empty())
636 return Error::success();
637
638 // Define the re-exports.
639 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
640}
641
642LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
643 : IPLS(std::move(IPLS)) {}
644
645void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
646
647LookupState::LookupState() = default;
648LookupState::LookupState(LookupState &&) = default;
649LookupState &LookupState::operator=(LookupState &&) = default;
650LookupState::~LookupState() = default;
651
652void LookupState::continueLookup(Error Err) {
653 assert(IPLS && "Cannot call continueLookup on empty LookupState");
654 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
655 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
656}
657
658DefinitionGenerator::~DefinitionGenerator() {
659 std::deque<LookupState> LookupsToFail;
660 {
661 std::lock_guard<std::mutex> Lock(M);
662 std::swap(x&: PendingLookups, y&: LookupsToFail);
663 InUse = false;
664 }
665
666 for (auto &LS : LookupsToFail)
667 LS.continueLookup(Err: make_error<StringError>(
668 Args: "Query waiting on DefinitionGenerator that was destroyed",
669 Args: inconvertibleErrorCode()));
670}
671
672JITDylib::~JITDylib() {
673 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
674}
675
676Error JITDylib::clear() {
677 std::vector<ResourceTrackerSP> TrackersToRemove;
678 ES.runSessionLocked(F: [&]() {
679 assert(State != Closed && "JD is defunct");
680 for (auto &KV : TrackerSymbols)
681 TrackersToRemove.push_back(x: KV.first);
682 TrackersToRemove.push_back(x: getDefaultResourceTracker());
683 });
684
685 Error Err = Error::success();
686 for (auto &RT : TrackersToRemove)
687 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
688 return Err;
689}
690
691ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
692 return ES.runSessionLocked(F: [this] {
693 assert(State != Closed && "JD is defunct");
694 if (!DefaultTracker)
695 DefaultTracker = new ResourceTracker(this);
696 return DefaultTracker;
697 });
698}
699
700ResourceTrackerSP JITDylib::createResourceTracker() {
701 return ES.runSessionLocked(F: [this] {
702 assert(State == Open && "JD is defunct");
703 ResourceTrackerSP RT = new ResourceTracker(this);
704 return RT;
705 });
706}
707
708void JITDylib::removeGenerator(DefinitionGenerator &G) {
709 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
710 // session lock (since it may have to send errors to pending queries).
711 std::shared_ptr<DefinitionGenerator> TmpDG;
712
713 ES.runSessionLocked(F: [&] {
714 assert(State == Open && "JD is defunct");
715 auto I = llvm::find_if(Range&: DefGenerators,
716 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
717 return H.get() == &G;
718 });
719 assert(I != DefGenerators.end() && "Generator not found");
720 TmpDG = std::move(*I);
721 DefGenerators.erase(position: I);
722 });
723}
724
725Expected<SymbolFlagsMap>
726JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
727 SymbolFlagsMap SymbolFlags) {
728
729 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
730 if (FromMR.RT->isDefunct())
731 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
732
733 std::vector<NonOwningSymbolStringPtr> AddedSyms;
734 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
735
736 for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end();
737 SFItr != SFEnd; ++SFItr) {
738
739 auto &Name = SFItr->first;
740 auto &Flags = SFItr->second;
741
742 auto EntryItr = Symbols.find(Val: Name);
743
744 // If the entry already exists...
745 if (EntryItr != Symbols.end()) {
746
747 // If this is a strong definition then error out.
748 if (!Flags.isWeak()) {
749 // Remove any symbols already added.
750 for (auto &S : AddedSyms)
751 Symbols.erase(I: Symbols.find_as(Val: S));
752
753 // FIXME: Return all duplicates.
754 return make_error<DuplicateDefinition>(Args: std::string(*Name));
755 }
756
757 // Otherwise just make a note to discard this symbol after the loop.
758 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
759 continue;
760 } else
761 EntryItr =
762 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
763
764 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
765 EntryItr->second.setState(SymbolState::Materializing);
766 }
767
768 // Remove any rejected weak definitions from the SymbolFlags map.
769 while (!RejectedWeakDefs.empty()) {
770 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
771 RejectedWeakDefs.pop_back();
772 }
773
774 return SymbolFlags;
775 });
776}
777
778Error JITDylib::replace(MaterializationResponsibility &FromMR,
779 std::unique_ptr<MaterializationUnit> MU) {
780 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
781 std::unique_ptr<MaterializationUnit> MustRunMU;
782 std::unique_ptr<MaterializationResponsibility> MustRunMR;
783
784 auto Err =
785 ES.runSessionLocked(F: [&, this]() -> Error {
786 if (FromMR.RT->isDefunct())
787 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
788
789#ifndef NDEBUG
790 for (auto &KV : MU->getSymbols()) {
791 auto SymI = Symbols.find(KV.first);
792 assert(SymI != Symbols.end() && "Replacing unknown symbol");
793 assert(SymI->second.getState() == SymbolState::Materializing &&
794 "Can not replace a symbol that ha is not materializing");
795 assert(!SymI->second.hasMaterializerAttached() &&
796 "Symbol should not have materializer attached already");
797 assert(UnmaterializedInfos.count(KV.first) == 0 &&
798 "Symbol being replaced should have no UnmaterializedInfo");
799 }
800#endif // NDEBUG
801
802 // If the tracker is defunct we need to bail out immediately.
803
804 // If any symbol has pending queries against it then we need to
805 // materialize MU immediately.
806 for (auto &KV : MU->getSymbols()) {
807 auto MII = MaterializingInfos.find(Val: KV.first);
808 if (MII != MaterializingInfos.end()) {
809 if (MII->second.hasQueriesPending()) {
810 MustRunMR = ES.createMaterializationResponsibility(
811 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
812 InitSymbol: std::move(MU->InitSymbol));
813 MustRunMU = std::move(MU);
814 return Error::success();
815 }
816 }
817 }
818
819 // Otherwise, make MU responsible for all the symbols.
820 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
821 args: FromMR.RT.get());
822 for (auto &KV : UMI->MU->getSymbols()) {
823 auto SymI = Symbols.find(Val: KV.first);
824 assert(SymI->second.getState() == SymbolState::Materializing &&
825 "Can not replace a symbol that is not materializing");
826 assert(!SymI->second.hasMaterializerAttached() &&
827 "Can not replace a symbol that has a materializer attached");
828 assert(UnmaterializedInfos.count(KV.first) == 0 &&
829 "Unexpected materializer entry in map");
830 SymI->second.setAddress(SymI->second.getAddress());
831 SymI->second.setMaterializerAttached(true);
832
833 auto &UMIEntry = UnmaterializedInfos[KV.first];
834 assert((!UMIEntry || !UMIEntry->MU) &&
835 "Replacing symbol with materializer still attached");
836 UMIEntry = UMI;
837 }
838
839 return Error::success();
840 });
841
842 if (Err)
843 return Err;
844
845 if (MustRunMU) {
846 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
847 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
848 args: std::move(MustRunMU), args: std::move(MustRunMR)));
849 } else {
850 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
851 }
852
853 return Error::success();
854}
855
856Expected<std::unique_ptr<MaterializationResponsibility>>
857JITDylib::delegate(MaterializationResponsibility &FromMR,
858 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
859
860 return ES.runSessionLocked(
861 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
862 if (FromMR.RT->isDefunct())
863 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
864
865 return ES.createMaterializationResponsibility(
866 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
867 });
868}
869
870SymbolNameSet
871JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
872 return ES.runSessionLocked(F: [&]() {
873 SymbolNameSet RequestedSymbols;
874
875 for (auto &KV : SymbolFlags) {
876 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
877 assert(Symbols.find(KV.first)->second.getState() !=
878 SymbolState::NeverSearched &&
879 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
880 "getRequestedSymbols can only be called for symbols that have "
881 "started materializing");
882 auto I = MaterializingInfos.find(Val: KV.first);
883 if (I == MaterializingInfos.end())
884 continue;
885
886 if (I->second.hasQueriesPending())
887 RequestedSymbols.insert(V: KV.first);
888 }
889
890 return RequestedSymbols;
891 });
892}
893
894Error JITDylib::resolve(MaterializationResponsibility &MR,
895 const SymbolMap &Resolved) {
896 AsynchronousSymbolQuerySet CompletedQueries;
897
898 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
899 if (MR.RT->isDefunct())
900 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
901
902 if (State != Open)
903 return make_error<StringError>(Args: "JITDylib " + getName() +
904 " is defunct",
905 Args: inconvertibleErrorCode());
906
907 struct WorklistEntry {
908 SymbolTable::iterator SymI;
909 ExecutorSymbolDef ResolvedSym;
910 };
911
912 SymbolNameSet SymbolsInErrorState;
913 std::vector<WorklistEntry> Worklist;
914 Worklist.reserve(n: Resolved.size());
915
916 // Build worklist and check for any symbols in the error state.
917 for (const auto &KV : Resolved) {
918
919 assert(!KV.second.getFlags().hasError() &&
920 "Resolution result can not have error flag set");
921
922 auto SymI = Symbols.find(Val: KV.first);
923
924 assert(SymI != Symbols.end() && "Symbol not found");
925 assert(!SymI->second.hasMaterializerAttached() &&
926 "Resolving symbol with materializer attached?");
927 assert(SymI->second.getState() == SymbolState::Materializing &&
928 "Symbol should be materializing");
929 assert(SymI->second.getAddress() == ExecutorAddr() &&
930 "Symbol has already been resolved");
931
932 if (SymI->second.getFlags().hasError())
933 SymbolsInErrorState.insert(V: KV.first);
934 else {
935 auto Flags = KV.second.getFlags();
936 Flags &= ~JITSymbolFlags::Common;
937 assert(Flags ==
938 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
939 "Resolved flags should match the declared flags");
940
941 Worklist.push_back(x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), Flags}});
942 }
943 }
944
945 // If any symbols were in the error state then bail out.
946 if (!SymbolsInErrorState.empty()) {
947 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
948 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
949 return make_error<FailedToMaterialize>(
950 Args: getExecutionSession().getSymbolStringPool(),
951 Args: std::move(FailedSymbolsDepMap));
952 }
953
954 while (!Worklist.empty()) {
955 auto SymI = Worklist.back().SymI;
956 auto ResolvedSym = Worklist.back().ResolvedSym;
957 Worklist.pop_back();
958
959 auto &Name = SymI->first;
960
961 // Resolved symbols can not be weak: discard the weak flag.
962 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
963 SymI->second.setAddress(ResolvedSym.getAddress());
964 SymI->second.setFlags(ResolvedFlags);
965 SymI->second.setState(SymbolState::Resolved);
966
967 auto MII = MaterializingInfos.find(Val: Name);
968 if (MII == MaterializingInfos.end())
969 continue;
970
971 auto &MI = MII->second;
972 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
973 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
974 Q->removeQueryDependence(JD&: *this, Name);
975 if (Q->isComplete())
976 CompletedQueries.insert(x: std::move(Q));
977 }
978 }
979
980 return Error::success();
981 }))
982 return Err;
983
984 // Otherwise notify all the completed queries.
985 for (auto &Q : CompletedQueries) {
986 assert(Q->isComplete() && "Q not completed");
987 Q->handleComplete(ES);
988 }
989
990 return Error::success();
991}
992
993void JITDylib::unlinkMaterializationResponsibility(
994 MaterializationResponsibility &MR) {
995 ES.runSessionLocked(F: [&]() {
996 auto I = TrackerMRs.find(Val: MR.RT.get());
997 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
998 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
999 I->second.erase(V: &MR);
1000 if (I->second.empty())
1001 TrackerMRs.erase(Val: MR.RT.get());
1002 });
1003}
1004
1005void JITDylib::shrinkMaterializationInfoMemory() {
1006 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1007 // memory since we may have long-lived JDs after linking is done.
1008
1009 if (UnmaterializedInfos.empty())
1010 UnmaterializedInfos.clear();
1011
1012 if (MaterializingInfos.empty())
1013 MaterializingInfos.clear();
1014}
1015
1016void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1017 bool LinkAgainstThisJITDylibFirst) {
1018 ES.runSessionLocked(F: [&]() {
1019 assert(State == Open && "JD is defunct");
1020 if (LinkAgainstThisJITDylibFirst) {
1021 LinkOrder.clear();
1022 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1023 LinkOrder.push_back(
1024 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1025 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1026 } else
1027 LinkOrder = std::move(NewLinkOrder);
1028 });
1029}
1030
1031void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1032 ES.runSessionLocked(F: [&]() {
1033 for (auto &KV : NewLinks) {
1034 // Skip elements of NewLinks that are already in the link order.
1035 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1036 continue;
1037
1038 LinkOrder.push_back(x: std::move(KV));
1039 }
1040 });
1041}
1042
1043void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1044 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1045}
1046
1047void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1048 JITDylibLookupFlags JDLookupFlags) {
1049 ES.runSessionLocked(F: [&]() {
1050 assert(State == Open && "JD is defunct");
1051 for (auto &KV : LinkOrder)
1052 if (KV.first == &OldJD) {
1053 KV = {&NewJD, JDLookupFlags};
1054 break;
1055 }
1056 });
1057}
1058
1059void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1060 ES.runSessionLocked(F: [&]() {
1061 assert(State == Open && "JD is defunct");
1062 auto I = llvm::find_if(Range&: LinkOrder,
1063 P: [&](const JITDylibSearchOrder::value_type &KV) {
1064 return KV.first == &JD;
1065 });
1066 if (I != LinkOrder.end())
1067 LinkOrder.erase(position: I);
1068 });
1069}
1070
1071Error JITDylib::remove(const SymbolNameSet &Names) {
1072 return ES.runSessionLocked(F: [&]() -> Error {
1073 assert(State == Open && "JD is defunct");
1074 using SymbolMaterializerItrPair =
1075 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1076 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1077 SymbolNameSet Missing;
1078 SymbolNameSet Materializing;
1079
1080 for (auto &Name : Names) {
1081 auto I = Symbols.find(Val: Name);
1082
1083 // Note symbol missing.
1084 if (I == Symbols.end()) {
1085 Missing.insert(V: Name);
1086 continue;
1087 }
1088
1089 // Note symbol materializing.
1090 if (I->second.getState() != SymbolState::NeverSearched &&
1091 I->second.getState() != SymbolState::Ready) {
1092 Materializing.insert(V: Name);
1093 continue;
1094 }
1095
1096 auto UMII = I->second.hasMaterializerAttached()
1097 ? UnmaterializedInfos.find(Val: Name)
1098 : UnmaterializedInfos.end();
1099 SymbolsToRemove.push_back(x: std::make_pair(x&: I, y&: UMII));
1100 }
1101
1102 // If any of the symbols are not defined, return an error.
1103 if (!Missing.empty())
1104 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1105 Args: std::move(Missing));
1106
1107 // If any of the symbols are currently materializing, return an error.
1108 if (!Materializing.empty())
1109 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1110 Args: std::move(Materializing));
1111
1112 // Remove the symbols.
1113 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1114 auto UMII = SymbolMaterializerItrPair.second;
1115
1116 // If there is a materializer attached, call discard.
1117 if (UMII != UnmaterializedInfos.end()) {
1118 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1119 UnmaterializedInfos.erase(I: UMII);
1120 }
1121
1122 auto SymI = SymbolMaterializerItrPair.first;
1123 Symbols.erase(I: SymI);
1124 }
1125
1126 shrinkMaterializationInfoMemory();
1127
1128 return Error::success();
1129 });
1130}
1131
1132void JITDylib::dump(raw_ostream &OS) {
1133 ES.runSessionLocked(F: [&, this]() {
1134 OS << "JITDylib \"" << getName() << "\" (ES: "
1135 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1136 << ", State = ";
1137 switch (State) {
1138 case Open:
1139 OS << "Open";
1140 break;
1141 case Closing:
1142 OS << "Closing";
1143 break;
1144 case Closed:
1145 OS << "Closed";
1146 break;
1147 }
1148 OS << ")\n";
1149 if (State == Closed)
1150 return;
1151 OS << "Link order: " << LinkOrder << "\n"
1152 << "Symbol table:\n";
1153
1154 // Sort symbols so we get a deterministic order and can check them in tests.
1155 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1156 for (auto &KV : Symbols)
1157 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1158 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1159 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1160
1161 for (auto &KV : SymbolsSorted) {
1162 OS << " \"" << *KV.first << "\": ";
1163 if (auto Addr = KV.second->getAddress())
1164 OS << Addr;
1165 else
1166 OS << "<not resolved> ";
1167
1168 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1169
1170 if (KV.second->hasMaterializerAttached()) {
1171 OS << " (Materializer ";
1172 auto I = UnmaterializedInfos.find(Val: KV.first);
1173 assert(I != UnmaterializedInfos.end() &&
1174 "Lazy symbol should have UnmaterializedInfo");
1175 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1176 } else
1177 OS << "\n";
1178 }
1179
1180 if (!MaterializingInfos.empty())
1181 OS << " MaterializingInfos entries:\n";
1182 for (auto &KV : MaterializingInfos) {
1183 OS << " \"" << *KV.first << "\":\n"
1184 << " " << KV.second.pendingQueries().size()
1185 << " pending queries: { ";
1186 for (const auto &Q : KV.second.pendingQueries())
1187 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1188 OS << "}\n Defining EDU: ";
1189 if (KV.second.DefiningEDU) {
1190 OS << KV.second.DefiningEDU.get() << " { ";
1191 for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
1192 OS << Name << " ";
1193 OS << "}\n";
1194 OS << " Dependencies:\n";
1195 if (!KV.second.DefiningEDU->Dependencies.empty()) {
1196 for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
1197 OS << " " << DepJD->getName() << ": [ ";
1198 for (auto &Dep : Deps)
1199 OS << Dep << " ";
1200 OS << "]\n";
1201 }
1202 } else
1203 OS << " none\n";
1204 } else
1205 OS << "none\n";
1206 OS << " Dependant EDUs:\n";
1207 if (!KV.second.DependantEDUs.empty()) {
1208 for (auto &DependantEDU : KV.second.DependantEDUs) {
1209 OS << " " << DependantEDU << ": "
1210 << DependantEDU->JD->getName() << " { ";
1211 for (auto &[Name, Flags] : DependantEDU->Symbols)
1212 OS << Name << " ";
1213 OS << "}\n";
1214 }
1215 } else
1216 OS << " none\n";
1217 assert((Symbols[KV.first].getState() != SymbolState::Ready ||
1218 (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
1219 !KV.second.DependantEDUs.empty())) &&
1220 "Stale materializing info entry");
1221 }
1222 });
1223}
1224
1225void JITDylib::MaterializingInfo::addQuery(
1226 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1227
1228 auto I = llvm::lower_bound(
1229 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1230 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1231 return V->getRequiredState() <= S;
1232 });
1233 PendingQueries.insert(position: I.base(), x: std::move(Q));
1234}
1235
1236void JITDylib::MaterializingInfo::removeQuery(
1237 const AsynchronousSymbolQuery &Q) {
1238 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1239 auto I = llvm::find_if(
1240 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1241 return V.get() == &Q;
1242 });
1243 assert(I != PendingQueries.end() &&
1244 "Query is not attached to this MaterializingInfo");
1245 PendingQueries.erase(position: I);
1246}
1247
1248JITDylib::AsynchronousSymbolQueryList
1249JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1250 AsynchronousSymbolQueryList Result;
1251 while (!PendingQueries.empty()) {
1252 if (PendingQueries.back()->getRequiredState() > RequiredState)
1253 break;
1254
1255 Result.push_back(x: std::move(PendingQueries.back()));
1256 PendingQueries.pop_back();
1257 }
1258
1259 return Result;
1260}
1261
1262JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1263 : JITLinkDylib(std::move(Name)), ES(ES) {
1264 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1265}
1266
1267std::pair<JITDylib::AsynchronousSymbolQuerySet,
1268 std::shared_ptr<SymbolDependenceMap>>
1269JITDylib::IL_removeTracker(ResourceTracker &RT) {
1270 // Note: Should be called under the session lock.
1271 assert(State != Closed && "JD is defunct");
1272
1273 SymbolNameVector SymbolsToRemove;
1274 SymbolNameVector SymbolsToFail;
1275
1276 if (&RT == DefaultTracker.get()) {
1277 SymbolNameSet TrackedSymbols;
1278 for (auto &KV : TrackerSymbols)
1279 for (auto &Sym : KV.second)
1280 TrackedSymbols.insert(V: Sym);
1281
1282 for (auto &KV : Symbols) {
1283 auto &Sym = KV.first;
1284 if (!TrackedSymbols.count(V: Sym))
1285 SymbolsToRemove.push_back(x: Sym);
1286 }
1287
1288 DefaultTracker.reset();
1289 } else {
1290 /// Check for a non-default tracker.
1291 auto I = TrackerSymbols.find(Val: &RT);
1292 if (I != TrackerSymbols.end()) {
1293 SymbolsToRemove = std::move(I->second);
1294 TrackerSymbols.erase(I);
1295 }
1296 // ... if not found this tracker was already defunct. Nothing to do.
1297 }
1298
1299 for (auto &Sym : SymbolsToRemove) {
1300 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1301
1302 // If there is a MaterializingInfo then collect any queries to fail.
1303 auto MII = MaterializingInfos.find(Val: Sym);
1304 if (MII != MaterializingInfos.end())
1305 SymbolsToFail.push_back(x: Sym);
1306 }
1307
1308 auto Result = ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1309
1310 // Removed symbols should be taken out of the table altogether.
1311 for (auto &Sym : SymbolsToRemove) {
1312 auto I = Symbols.find(Val: Sym);
1313 assert(I != Symbols.end() && "Symbol not present in table");
1314
1315 // Remove Materializer if present.
1316 if (I->second.hasMaterializerAttached()) {
1317 // FIXME: Should this discard the symbols?
1318 UnmaterializedInfos.erase(Val: Sym);
1319 } else {
1320 assert(!UnmaterializedInfos.count(Sym) &&
1321 "Symbol has materializer attached");
1322 }
1323
1324 Symbols.erase(I);
1325 }
1326
1327 shrinkMaterializationInfoMemory();
1328
1329 return Result;
1330}
1331
1332void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1333 assert(State != Closed && "JD is defunct");
1334 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1335 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1336 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1337
1338 // Update trackers for any not-yet materialized units.
1339 for (auto &KV : UnmaterializedInfos) {
1340 if (KV.second->RT == &SrcRT)
1341 KV.second->RT = &DstRT;
1342 }
1343
1344 // Update trackers for any active materialization responsibilities.
1345 {
1346 auto I = TrackerMRs.find(Val: &SrcRT);
1347 if (I != TrackerMRs.end()) {
1348 auto &SrcMRs = I->second;
1349 auto &DstMRs = TrackerMRs[&DstRT];
1350 for (auto *MR : SrcMRs)
1351 MR->RT = &DstRT;
1352 if (DstMRs.empty())
1353 DstMRs = std::move(SrcMRs);
1354 else
1355 for (auto *MR : SrcMRs)
1356 DstMRs.insert(V: MR);
1357 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1358 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1359 TrackerMRs.erase(Val: &SrcRT);
1360 }
1361 }
1362
1363 // If we're transfering to the default tracker we just need to delete the
1364 // tracked symbols for the source tracker.
1365 if (&DstRT == DefaultTracker.get()) {
1366 TrackerSymbols.erase(Val: &SrcRT);
1367 return;
1368 }
1369
1370 // If we're transferring from the default tracker we need to find all
1371 // currently untracked symbols.
1372 if (&SrcRT == DefaultTracker.get()) {
1373 assert(!TrackerSymbols.count(&SrcRT) &&
1374 "Default tracker should not appear in TrackerSymbols");
1375
1376 SymbolNameVector SymbolsToTrack;
1377
1378 SymbolNameSet CurrentlyTrackedSymbols;
1379 for (auto &KV : TrackerSymbols)
1380 for (auto &Sym : KV.second)
1381 CurrentlyTrackedSymbols.insert(V: Sym);
1382
1383 for (auto &KV : Symbols) {
1384 auto &Sym = KV.first;
1385 if (!CurrentlyTrackedSymbols.count(V: Sym))
1386 SymbolsToTrack.push_back(x: Sym);
1387 }
1388
1389 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1390 return;
1391 }
1392
1393 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1394
1395 // Finally if neither SrtRT or DstRT are the default tracker then
1396 // just append DstRT's tracked symbols to SrtRT's.
1397 auto SI = TrackerSymbols.find(Val: &SrcRT);
1398 if (SI == TrackerSymbols.end())
1399 return;
1400
1401 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1402 for (auto &Sym : SI->second)
1403 DstTrackedSymbols.push_back(x: std::move(Sym));
1404 TrackerSymbols.erase(I: SI);
1405}
1406
1407Error JITDylib::defineImpl(MaterializationUnit &MU) {
1408 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1409
1410 SymbolNameSet Duplicates;
1411 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1412 std::vector<SymbolStringPtr> MUDefsOverridden;
1413
1414 for (const auto &KV : MU.getSymbols()) {
1415 auto I = Symbols.find(Val: KV.first);
1416
1417 if (I != Symbols.end()) {
1418 if (KV.second.isStrong()) {
1419 if (I->second.getFlags().isStrong() ||
1420 I->second.getState() > SymbolState::NeverSearched)
1421 Duplicates.insert(V: KV.first);
1422 else {
1423 assert(I->second.getState() == SymbolState::NeverSearched &&
1424 "Overridden existing def should be in the never-searched "
1425 "state");
1426 ExistingDefsOverridden.push_back(x: KV.first);
1427 }
1428 } else
1429 MUDefsOverridden.push_back(x: KV.first);
1430 }
1431 }
1432
1433 // If there were any duplicate definitions then bail out.
1434 if (!Duplicates.empty()) {
1435 LLVM_DEBUG(
1436 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1437 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()));
1438 }
1439
1440 // Discard any overridden defs in this MU.
1441 LLVM_DEBUG({
1442 if (!MUDefsOverridden.empty())
1443 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1444 });
1445 for (auto &S : MUDefsOverridden)
1446 MU.doDiscard(JD: *this, Name: S);
1447
1448 // Discard existing overridden defs.
1449 LLVM_DEBUG({
1450 if (!ExistingDefsOverridden.empty())
1451 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1452 << "\n";
1453 });
1454 for (auto &S : ExistingDefsOverridden) {
1455
1456 auto UMII = UnmaterializedInfos.find(Val: S);
1457 assert(UMII != UnmaterializedInfos.end() &&
1458 "Overridden existing def should have an UnmaterializedInfo");
1459 UMII->second->MU->doDiscard(JD: *this, Name: S);
1460 }
1461
1462 // Finally, add the defs from this MU.
1463 for (auto &KV : MU.getSymbols()) {
1464 auto &SymEntry = Symbols[KV.first];
1465 SymEntry.setFlags(KV.second);
1466 SymEntry.setState(SymbolState::NeverSearched);
1467 SymEntry.setMaterializerAttached(true);
1468 }
1469
1470 return Error::success();
1471}
1472
1473void JITDylib::installMaterializationUnit(
1474 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1475
1476 /// defineImpl succeeded.
1477 if (&RT != DefaultTracker.get()) {
1478 auto &TS = TrackerSymbols[&RT];
1479 TS.reserve(n: TS.size() + MU->getSymbols().size());
1480 for (auto &KV : MU->getSymbols())
1481 TS.push_back(x: KV.first);
1482 }
1483
1484 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1485 for (auto &KV : UMI->MU->getSymbols())
1486 UnmaterializedInfos[KV.first] = UMI;
1487}
1488
1489void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1490 const SymbolNameSet &QuerySymbols) {
1491 for (auto &QuerySymbol : QuerySymbols) {
1492 assert(MaterializingInfos.count(QuerySymbol) &&
1493 "QuerySymbol does not have MaterializingInfo");
1494 auto &MI = MaterializingInfos[QuerySymbol];
1495 MI.removeQuery(Q);
1496 }
1497}
1498
1499Platform::~Platform() = default;
1500
1501Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1502 ExecutionSession &ES,
1503 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1504
1505 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1506 Error CompoundErr = Error::success();
1507 std::mutex LookupMutex;
1508 std::condition_variable CV;
1509 uint64_t Count = InitSyms.size();
1510
1511 LLVM_DEBUG({
1512 dbgs() << "Issuing init-symbol lookup:\n";
1513 for (auto &KV : InitSyms)
1514 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1515 });
1516
1517 for (auto &KV : InitSyms) {
1518 auto *JD = KV.first;
1519 auto Names = std::move(KV.second);
1520 ES.lookup(
1521 K: LookupKind::Static,
1522 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1523 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1524 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1525 {
1526 std::lock_guard<std::mutex> Lock(LookupMutex);
1527 --Count;
1528 if (Result) {
1529 assert(!CompoundResult.count(JD) &&
1530 "Duplicate JITDylib in lookup?");
1531 CompoundResult[JD] = std::move(*Result);
1532 } else
1533 CompoundErr =
1534 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1535 }
1536 CV.notify_one();
1537 },
1538 RegisterDependencies: NoDependenciesToRegister);
1539 }
1540
1541 std::unique_lock<std::mutex> Lock(LookupMutex);
1542 CV.wait(lock&: Lock, p: [&] { return Count == 0 || CompoundErr; });
1543
1544 if (CompoundErr)
1545 return std::move(CompoundErr);
1546
1547 return std::move(CompoundResult);
1548}
1549
1550void Platform::lookupInitSymbolsAsync(
1551 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1552 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1553
1554 class TriggerOnComplete {
1555 public:
1556 using OnCompleteFn = unique_function<void(Error)>;
1557 TriggerOnComplete(OnCompleteFn OnComplete)
1558 : OnComplete(std::move(OnComplete)) {}
1559 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1560 void reportResult(Error Err) {
1561 std::lock_guard<std::mutex> Lock(ResultMutex);
1562 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1563 }
1564
1565 private:
1566 std::mutex ResultMutex;
1567 Error LookupResult{Error::success()};
1568 OnCompleteFn OnComplete;
1569 };
1570
1571 LLVM_DEBUG({
1572 dbgs() << "Issuing init-symbol lookup:\n";
1573 for (auto &KV : InitSyms)
1574 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1575 });
1576
1577 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1578
1579 for (auto &KV : InitSyms) {
1580 auto *JD = KV.first;
1581 auto Names = std::move(KV.second);
1582 ES.lookup(
1583 K: LookupKind::Static,
1584 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1585 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1586 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1587 TOC->reportResult(Err: Result.takeError());
1588 },
1589 RegisterDependencies: NoDependenciesToRegister);
1590 }
1591}
1592
1593void MaterializationTask::printDescription(raw_ostream &OS) {
1594 OS << "Materialization task: " << MU->getName() << " in "
1595 << MR->getTargetJITDylib().getName();
1596}
1597
1598void MaterializationTask::run() { MU->materialize(R: std::move(MR)); }
1599
1600void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1601
1602void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1603
1604ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1605 : EPC(std::move(EPC)) {
1606 // Associated EPC and this.
1607 this->EPC->ES = this;
1608}
1609
1610ExecutionSession::~ExecutionSession() {
1611 // You must call endSession prior to destroying the session.
1612 assert(!SessionOpen &&
1613 "Session still open. Did you forget to call endSession?");
1614}
1615
1616Error ExecutionSession::endSession() {
1617 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1618
1619 auto JDsToRemove = runSessionLocked(F: [&] {
1620
1621#ifdef EXPENSIVE_CHECKS
1622 verifySessionState("Entering ExecutionSession::endSession");
1623#endif
1624
1625 SessionOpen = false;
1626 return JDs;
1627 });
1628
1629 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1630
1631 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1632
1633 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1634
1635 return Err;
1636}
1637
1638void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1639 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1640}
1641
1642void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1643 runSessionLocked(F: [&] {
1644 assert(!ResourceManagers.empty() && "No managers registered");
1645 if (ResourceManagers.back() == &RM)
1646 ResourceManagers.pop_back();
1647 else {
1648 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1649 assert(I != ResourceManagers.end() && "RM not registered");
1650 ResourceManagers.erase(position: I);
1651 }
1652 });
1653}
1654
1655JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1656 return runSessionLocked(F: [&, this]() -> JITDylib * {
1657 for (auto &JD : JDs)
1658 if (JD->getName() == Name)
1659 return JD.get();
1660 return nullptr;
1661 });
1662}
1663
1664JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1665 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1666 return runSessionLocked(F: [&, this]() -> JITDylib & {
1667 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1668 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1669 return *JDs.back();
1670 });
1671}
1672
1673Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1674 auto &JD = createBareJITDylib(Name);
1675 if (P)
1676 if (auto Err = P->setupJITDylib(JD))
1677 return std::move(Err);
1678 return JD;
1679}
1680
1681Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1682 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1683 runSessionLocked(F: [&] {
1684 for (auto &JD : JDsToRemove) {
1685 assert(JD->State == JITDylib::Open && "JD already closed");
1686 JD->State = JITDylib::Closing;
1687 auto I = llvm::find(Range&: JDs, Val: JD);
1688 assert(I != JDs.end() && "JD does not appear in session JDs");
1689 JDs.erase(position: I);
1690 }
1691 });
1692
1693 // Clear JITDylibs and notify the platform.
1694 Error Err = Error::success();
1695 for (auto JD : JDsToRemove) {
1696 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1697 if (P)
1698 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1699 }
1700
1701 // Set JD to closed state. Clear remaining data structures.
1702 runSessionLocked(F: [&] {
1703 for (auto &JD : JDsToRemove) {
1704 assert(JD->State == JITDylib::Closing && "JD should be closing");
1705 JD->State = JITDylib::Closed;
1706 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1707 assert(JD->UnmaterializedInfos.empty() &&
1708 "JD.UnmaterializedInfos is not empty after clear");
1709 assert(JD->MaterializingInfos.empty() &&
1710 "JD.MaterializingInfos is not empty after clear");
1711 assert(JD->TrackerSymbols.empty() &&
1712 "TrackerSymbols is not empty after clear");
1713 JD->DefGenerators.clear();
1714 JD->LinkOrder.clear();
1715 }
1716 });
1717
1718 return Err;
1719}
1720
1721Expected<std::vector<JITDylibSP>>
1722JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1723 if (JDs.empty())
1724 return std::vector<JITDylibSP>();
1725
1726 auto &ES = JDs.front()->getExecutionSession();
1727 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1728 DenseSet<JITDylib *> Visited;
1729 std::vector<JITDylibSP> Result;
1730
1731 for (auto &JD : JDs) {
1732
1733 if (JD->State != Open)
1734 return make_error<StringError>(
1735 Args: "Error building link order: " + JD->getName() + " is defunct",
1736 Args: inconvertibleErrorCode());
1737 if (Visited.count(V: JD.get()))
1738 continue;
1739
1740 SmallVector<JITDylibSP, 64> WorkStack;
1741 WorkStack.push_back(Elt: JD);
1742 Visited.insert(V: JD.get());
1743
1744 while (!WorkStack.empty()) {
1745 Result.push_back(x: std::move(WorkStack.back()));
1746 WorkStack.pop_back();
1747
1748 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1749 auto &JD = *KV.first;
1750 if (!Visited.insert(V: &JD).second)
1751 continue;
1752 WorkStack.push_back(Elt: &JD);
1753 }
1754 }
1755 }
1756 return Result;
1757 });
1758}
1759
1760Expected<std::vector<JITDylibSP>>
1761JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1762 auto Result = getDFSLinkOrder(JDs);
1763 if (Result)
1764 std::reverse(first: Result->begin(), last: Result->end());
1765 return Result;
1766}
1767
1768Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1769 return getDFSLinkOrder(JDs: {this});
1770}
1771
1772Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1773 return getReverseDFSLinkOrder(JDs: {this});
1774}
1775
1776void ExecutionSession::lookupFlags(
1777 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1778 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1779
1780 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1781 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1782 args: std::move(OnComplete)),
1783 Err: Error::success());
1784}
1785
1786Expected<SymbolFlagsMap>
1787ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1788 SymbolLookupSet LookupSet) {
1789
1790 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1791 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1792 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1793 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1794 ResultP.set_value(std::move(Result));
1795 }),
1796 Err: Error::success());
1797
1798 auto ResultF = ResultP.get_future();
1799 return ResultF.get();
1800}
1801
1802void ExecutionSession::lookup(
1803 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1804 SymbolLookupSet Symbols, SymbolState RequiredState,
1805 SymbolsResolvedCallback NotifyComplete,
1806 RegisterDependenciesFunction RegisterDependencies) {
1807
1808 LLVM_DEBUG({
1809 runSessionLocked([&]() {
1810 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1811 << " (required state: " << RequiredState << ")\n";
1812 });
1813 });
1814
1815 // lookup can be re-entered recursively if running on a single thread. Run any
1816 // outstanding MUs in case this query depends on them, otherwise this lookup
1817 // will starve waiting for a result from an MU that is stuck in the queue.
1818 dispatchOutstandingMUs();
1819
1820 auto Unresolved = std::move(Symbols);
1821 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1822 args: std::move(NotifyComplete));
1823
1824 auto IPLS = std::make_unique<InProgressFullLookupState>(
1825 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1826 args: std::move(RegisterDependencies));
1827
1828 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1829}
1830
1831Expected<SymbolMap>
1832ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1833 SymbolLookupSet Symbols, LookupKind K,
1834 SymbolState RequiredState,
1835 RegisterDependenciesFunction RegisterDependencies) {
1836#if LLVM_ENABLE_THREADS
1837 // In the threaded case we use promises to return the results.
1838 std::promise<SymbolMap> PromisedResult;
1839 Error ResolutionError = Error::success();
1840
1841 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1842 if (R)
1843 PromisedResult.set_value(std::move(*R));
1844 else {
1845 ErrorAsOutParameter _(&ResolutionError);
1846 ResolutionError = R.takeError();
1847 PromisedResult.set_value(SymbolMap());
1848 }
1849 };
1850
1851#else
1852 SymbolMap Result;
1853 Error ResolutionError = Error::success();
1854
1855 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1856 ErrorAsOutParameter _(&ResolutionError);
1857 if (R)
1858 Result = std::move(*R);
1859 else
1860 ResolutionError = R.takeError();
1861 };
1862#endif
1863
1864 // Perform the asynchronous lookup.
1865 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState, NotifyComplete,
1866 RegisterDependencies);
1867
1868#if LLVM_ENABLE_THREADS
1869 auto ResultFuture = PromisedResult.get_future();
1870 auto Result = ResultFuture.get();
1871
1872 if (ResolutionError)
1873 return std::move(ResolutionError);
1874
1875 return std::move(Result);
1876
1877#else
1878 if (ResolutionError)
1879 return std::move(ResolutionError);
1880
1881 return Result;
1882#endif
1883}
1884
1885Expected<ExecutorSymbolDef>
1886ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1887 SymbolStringPtr Name, SymbolState RequiredState) {
1888 SymbolLookupSet Names({Name});
1889
1890 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1891 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1892 assert(ResultMap->size() == 1 && "Unexpected number of results");
1893 assert(ResultMap->count(Name) && "Missing result for symbol");
1894 return std::move(ResultMap->begin()->second);
1895 } else
1896 return ResultMap.takeError();
1897}
1898
1899Expected<ExecutorSymbolDef>
1900ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1901 SymbolState RequiredState) {
1902 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1903}
1904
1905Expected<ExecutorSymbolDef>
1906ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1907 SymbolState RequiredState) {
1908 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1909}
1910
1911Error ExecutionSession::registerJITDispatchHandlers(
1912 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1913
1914 auto TagAddrs = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1915 Symbols: SymbolLookupSet::fromMapKeys(
1916 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1917 if (!TagAddrs)
1918 return TagAddrs.takeError();
1919
1920 // Associate tag addresses with implementations.
1921 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1922 for (auto &KV : *TagAddrs) {
1923 auto TagAddr = KV.second.getAddress();
1924 if (JITDispatchHandlers.count(Val: TagAddr))
1925 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x16}", Vals&: TagAddr) +
1926 " (for " + *KV.first +
1927 ") already registered",
1928 Args: inconvertibleErrorCode());
1929 auto I = WFs.find(Val: KV.first);
1930 assert(I != WFs.end() && I->second &&
1931 "JITDispatchHandler implementation missing");
1932 JITDispatchHandlers[KV.second.getAddress()] =
1933 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1934 LLVM_DEBUG({
1935 dbgs() << "Associated function tag \"" << *KV.first << "\" ("
1936 << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
1937 });
1938 }
1939 return Error::success();
1940}
1941
1942void ExecutionSession::runJITDispatchHandler(SendResultFunction SendResult,
1943 ExecutorAddr HandlerFnTagAddr,
1944 ArrayRef<char> ArgBuffer) {
1945
1946 std::shared_ptr<JITDispatchHandlerFunction> F;
1947 {
1948 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1949 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1950 if (I != JITDispatchHandlers.end())
1951 F = I->second;
1952 }
1953
1954 if (F)
1955 (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size());
1956 else
1957 SendResult(shared::WrapperFunctionResult::createOutOfBandError(
1958 Msg: ("No function registered for tag " +
1959 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1960 .str()));
1961}
1962
1963void ExecutionSession::dump(raw_ostream &OS) {
1964 runSessionLocked(F: [this, &OS]() {
1965 for (auto &JD : JDs)
1966 JD->dump(OS);
1967 });
1968}
1969
1970#ifdef EXPENSIVE_CHECKS
1971bool ExecutionSession::verifySessionState(Twine Phase) {
1972 return runSessionLocked([&]() {
1973 bool AllOk = true;
1974
1975 // We'll collect these and verify them later to avoid redundant checks.
1976 DenseSet<JITDylib::EmissionDepUnit *> EDUsToCheck;
1977
1978 for (auto &JD : JDs) {
1979
1980 auto LogFailure = [&]() -> raw_fd_ostream & {
1981 auto &Stream = errs();
1982 if (AllOk)
1983 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1984 << "\n";
1985 Stream << " In JITDylib " << JD->getName() << ", ";
1986 AllOk = false;
1987 return Stream;
1988 };
1989
1990 if (JD->State != JITDylib::Open) {
1991 LogFailure()
1992 << "state is not Open, but JD is in ExecutionSession list.";
1993 }
1994
1995 // Check symbol table.
1996 // 1. If the entry state isn't resolved then check that no address has
1997 // been set.
1998 // 2. Check that if the hasMaterializerAttached flag is set then there is
1999 // an UnmaterializedInfo entry, and vice-versa.
2000 for (auto &[Sym, Entry] : JD->Symbols) {
2001 // Check that unresolved symbols have null addresses.
2002 if (Entry.getState() < SymbolState::Resolved) {
2003 if (Entry.getAddress()) {
2004 LogFailure() << "symbol " << Sym << " has state "
2005 << Entry.getState()
2006 << " (not-yet-resolved) but non-null address "
2007 << Entry.getAddress() << ".\n";
2008 }
2009 }
2010
2011 // Check that the hasMaterializerAttached flag is correct.
2012 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2013 if (Entry.hasMaterializerAttached()) {
2014 if (UMIItr == JD->UnmaterializedInfos.end()) {
2015 LogFailure() << "symbol " << Sym
2016 << " entry claims materializer attached, but "
2017 "UnmaterializedInfos has no corresponding entry.\n";
2018 }
2019 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2020 LogFailure()
2021 << "symbol " << Sym
2022 << " entry claims no materializer attached, but "
2023 "UnmaterializedInfos has an unexpected entry for it.\n";
2024 }
2025 }
2026
2027 // Check that every UnmaterializedInfo entry has a corresponding entry
2028 // in the Symbols table.
2029 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2030 auto SymItr = JD->Symbols.find(Sym);
2031 if (SymItr == JD->Symbols.end()) {
2032 LogFailure()
2033 << "symbol " << Sym
2034 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2035 }
2036 }
2037
2038 // Check consistency of the MaterializingInfos table.
2039 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2040
2041 auto SymItr = JD->Symbols.find(Sym);
2042 if (SymItr == JD->Symbols.end()) {
2043 // If there's no Symbols entry for this MaterializingInfos entry then
2044 // report that.
2045 LogFailure()
2046 << "symbol " << Sym
2047 << " has MaterializingInfos entry, but no Symbols entry.\n";
2048 } else {
2049 // Otherwise check consistency between Symbols and MaterializingInfos.
2050
2051 // Ready symbols should not have MaterializingInfos.
2052 if (SymItr->second.getState() == SymbolState::Ready) {
2053 LogFailure()
2054 << "symbol " << Sym
2055 << " is in Ready state, should not have MaterializingInfo.\n";
2056 }
2057
2058 // Pending queries should be for subsequent states.
2059 auto CurState = static_cast<SymbolState>(
2060 static_cast<std::underlying_type_t<SymbolState>>(
2061 SymItr->second.getState()) + 1);
2062 for (auto &Q : MII.PendingQueries) {
2063 if (Q->getRequiredState() != CurState) {
2064 if (Q->getRequiredState() > CurState)
2065 CurState = Q->getRequiredState();
2066 else
2067 LogFailure() << "symbol " << Sym
2068 << " has stale or misordered queries.\n";
2069 }
2070 }
2071
2072 // If there's a DefiningEDU then check that...
2073 // 1. The JD matches.
2074 // 2. The symbol is in the EDU's Symbols map.
2075 // 3. The symbol table entry is in the Emitted state.
2076 if (MII.DefiningEDU) {
2077
2078 EDUsToCheck.insert(MII.DefiningEDU.get());
2079
2080 if (MII.DefiningEDU->JD != JD.get()) {
2081 LogFailure() << "symbol " << Sym
2082 << " has DefiningEDU with incorrect JD"
2083 << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
2084 ? " (JD not currently in ExecutionSession"
2085 : "")
2086 << "\n";
2087 }
2088
2089 if (SymItr->second.getState() != SymbolState::Emitted) {
2090 LogFailure()
2091 << "symbol " << Sym
2092 << " has DefiningEDU, but is not in Emitted state.\n";
2093 }
2094 }
2095
2096 // Check that JDs for any DependantEDUs are also in the session --
2097 // that guarantees that we'll also visit them during this loop.
2098 for (auto &DepEDU : MII.DependantEDUs) {
2099 if (!llvm::is_contained(JDs, DepEDU->JD)) {
2100 LogFailure() << "symbol " << Sym << " has DependantEDU "
2101 << (void *)DepEDU << " with JD (" << DepEDU->JD
2102 << ") that isn't in ExecutionSession.\n";
2103 }
2104 }
2105 }
2106 }
2107 }
2108
2109 // Check EDUs.
2110 for (auto *EDU : EDUsToCheck) {
2111 assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
2112
2113 auto LogFailure = [&]() -> raw_fd_ostream & {
2114 AllOk = false;
2115 auto &Stream = errs();
2116 Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
2117 for (auto &[Sym, Flags] : EDU->Symbols)
2118 Stream << Sym << " ";
2119 Stream << "}, ";
2120 return Stream;
2121 };
2122
2123 if (EDU->Symbols.empty())
2124 LogFailure() << "no symbols defined.\n";
2125 else {
2126 for (auto &[Sym, Flags] : EDU->Symbols) {
2127 if (!Sym)
2128 LogFailure() << "null symbol defined.\n";
2129 else {
2130 if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
2131 LogFailure() << "symbol " << Sym
2132 << " isn't present in JD's symbol table.\n";
2133 }
2134 }
2135 }
2136 }
2137
2138 for (auto &[DepJD, Symbols] : EDU->Dependencies) {
2139 if (!llvm::is_contained(JDs, DepJD)) {
2140 LogFailure() << "dependant symbols listed for JD that isn't in "
2141 "ExecutionSession.\n";
2142 } else {
2143 for (auto &DepSym : Symbols) {
2144 if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
2145 LogFailure()
2146 << "dependant symbol " << DepSym
2147 << " does not appear in symbol table for dependant JD "
2148 << DepJD->getName() << ".\n";
2149 }
2150 }
2151 }
2152 }
2153 }
2154
2155 return AllOk;
2156 });
2157}
2158#endif // EXPENSIVE_CHECKS
2159
2160void ExecutionSession::dispatchOutstandingMUs() {
2161 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2162 while (true) {
2163 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2164 std::unique_ptr<MaterializationResponsibility>>>
2165 JMU;
2166
2167 {
2168 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2169 if (!OutstandingMUs.empty()) {
2170 JMU.emplace(args: std::move(OutstandingMUs.back()));
2171 OutstandingMUs.pop_back();
2172 }
2173 }
2174
2175 if (!JMU)
2176 break;
2177
2178 assert(JMU->first && "No MU?");
2179 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2180 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2181 args: std::move(JMU->second)));
2182 }
2183 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2184}
2185
2186Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2187 LLVM_DEBUG({
2188 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2189 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2190 });
2191 std::vector<ResourceManager *> CurrentResourceManagers;
2192
2193 JITDylib::AsynchronousSymbolQuerySet QueriesToFail;
2194 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
2195
2196 runSessionLocked(F: [&] {
2197 CurrentResourceManagers = ResourceManagers;
2198 RT.makeDefunct();
2199 std::tie(args&: QueriesToFail, args&: FailedSymbols) =
2200 RT.getJITDylib().IL_removeTracker(RT);
2201 });
2202
2203 Error Err = Error::success();
2204
2205 auto &JD = RT.getJITDylib();
2206 for (auto *L : reverse(C&: CurrentResourceManagers))
2207 Err = joinErrors(E1: std::move(Err),
2208 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2209
2210 for (auto &Q : QueriesToFail)
2211 Q->handleFailed(
2212 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
2213
2214 return Err;
2215}
2216
2217void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2218 ResourceTracker &SrcRT) {
2219 LLVM_DEBUG({
2220 dbgs() << "In " << SrcRT.getJITDylib().getName()
2221 << " transfering resources from tracker "
2222 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2223 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2224 });
2225
2226 // No-op transfers are allowed and do not invalidate the source.
2227 if (&DstRT == &SrcRT)
2228 return;
2229
2230 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2231 "Can't transfer resources between JITDylibs");
2232 runSessionLocked(F: [&]() {
2233 SrcRT.makeDefunct();
2234 auto &JD = DstRT.getJITDylib();
2235 JD.transferTracker(DstRT, SrcRT);
2236 for (auto *L : reverse(C&: ResourceManagers))
2237 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2238 SrcK: SrcRT.getKeyUnsafe());
2239 });
2240}
2241
2242void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2243 runSessionLocked(F: [&]() {
2244 LLVM_DEBUG({
2245 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2246 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2247 });
2248 if (!RT.isDefunct())
2249 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2250 SrcRT&: RT);
2251 });
2252}
2253
2254Error ExecutionSession::IL_updateCandidatesFor(
2255 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2256 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2257 return Candidates.forEachWithRemoval(
2258 Body: [&](const SymbolStringPtr &Name,
2259 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2260 /// Search for the symbol. If not found then continue without
2261 /// removal.
2262 auto SymI = JD.Symbols.find(Val: Name);
2263 if (SymI == JD.Symbols.end())
2264 return false;
2265
2266 // If this is a non-exported symbol and we're matching exported
2267 // symbols only then remove this symbol from the candidates list.
2268 //
2269 // If we're tracking non-candidates then add this to the non-candidate
2270 // list.
2271 if (!SymI->second.getFlags().isExported() &&
2272 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2273 if (NonCandidates)
2274 NonCandidates->add(Name, Flags: SymLookupFlags);
2275 return true;
2276 }
2277
2278 // If we match against a materialization-side-effects only symbol
2279 // then make sure it is weakly-referenced. Otherwise bail out with
2280 // an error.
2281 // FIXME: Use a "materialization-side-effects-only symbols must be
2282 // weakly referenced" specific error here to reduce confusion.
2283 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2284 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2285 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2286 Args: SymbolNameVector({Name}));
2287
2288 // If we matched against this symbol but it is in the error state
2289 // then bail out and treat it as a failure to materialize.
2290 if (SymI->second.getFlags().hasError()) {
2291 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2292 (*FailedSymbolsMap)[&JD] = {Name};
2293 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2294 Args: std::move(FailedSymbolsMap));
2295 }
2296
2297 // Otherwise this is a match. Remove it from the candidate set.
2298 return true;
2299 });
2300}
2301
2302void ExecutionSession::OL_resumeLookupAfterGeneration(
2303 InProgressLookupState &IPLS) {
2304
2305 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2306 "Should not be called for not-in-generator lookups");
2307 IPLS.GenState = InProgressLookupState::NotInGenerator;
2308
2309 LookupState LS;
2310
2311 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2312 IPLS.CurDefGeneratorStack.pop_back();
2313 std::lock_guard<std::mutex> Lock(DG->M);
2314
2315 // If there are no pending lookups then mark the generator as free and
2316 // return.
2317 if (DG->PendingLookups.empty()) {
2318 DG->InUse = false;
2319 return;
2320 }
2321
2322 // Otherwise resume the next lookup.
2323 LS = std::move(DG->PendingLookups.front());
2324 DG->PendingLookups.pop_front();
2325 }
2326
2327 if (LS.IPLS) {
2328 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2329 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2330 }
2331}
2332
2333void ExecutionSession::OL_applyQueryPhase1(
2334 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2335
2336 LLVM_DEBUG({
2337 dbgs() << "Entering OL_applyQueryPhase1:\n"
2338 << " Lookup kind: " << IPLS->K << "\n"
2339 << " Search order: " << IPLS->SearchOrder
2340 << ", Current index = " << IPLS->CurSearchOrderIndex
2341 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2342 << " Lookup set: " << IPLS->LookupSet << "\n"
2343 << " Definition generator candidates: "
2344 << IPLS->DefGeneratorCandidates << "\n"
2345 << " Definition generator non-candidates: "
2346 << IPLS->DefGeneratorNonCandidates << "\n";
2347 });
2348
2349 if (IPLS->GenState == InProgressLookupState::InGenerator)
2350 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2351
2352 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2353 "Lookup should not be in InGenerator state here");
2354
2355 // FIXME: We should attach the query as we go: This provides a result in a
2356 // single pass in the common case where all symbols have already reached the
2357 // required state. The query could be detached again in the 'fail' method on
2358 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2359
2360 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2361
2362 // If we've been handed an error or received one back from a generator then
2363 // fail the query. We don't need to unlink: At this stage the query hasn't
2364 // actually been lodged.
2365 if (Err)
2366 return IPLS->fail(Err: std::move(Err));
2367
2368 // Get the next JITDylib and lookup flags.
2369 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2370 auto &JD = *KV.first;
2371 auto JDLookupFlags = KV.second;
2372
2373 LLVM_DEBUG({
2374 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2375 << ") with lookup set " << IPLS->LookupSet << ":\n";
2376 });
2377
2378 // If we've just reached a new JITDylib then perform some setup.
2379 if (IPLS->NewJITDylib) {
2380 // Add any non-candidates from the last JITDylib (if any) back on to the
2381 // list of definition candidates for this JITDylib, reset definition
2382 // non-candidates to the empty set.
2383 SymbolLookupSet Tmp;
2384 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2385 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2386
2387 LLVM_DEBUG({
2388 dbgs() << " First time visiting " << JD.getName()
2389 << ", resetting candidate sets and building generator stack\n";
2390 });
2391
2392 // Build the definition generator stack for this JITDylib.
2393 runSessionLocked(F: [&] {
2394 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2395 for (auto &DG : reverse(C&: JD.DefGenerators))
2396 IPLS->CurDefGeneratorStack.push_back(x: DG);
2397 });
2398
2399 // Flag that we've done our initialization.
2400 IPLS->NewJITDylib = false;
2401 }
2402
2403 // Remove any generation candidates that are already defined (and match) in
2404 // this JITDylib.
2405 runSessionLocked(F: [&] {
2406 // Update the list of candidates (and non-candidates) for definition
2407 // generation.
2408 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2409 Err = IL_updateCandidatesFor(
2410 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2411 NonCandidates: JD.DefGenerators.empty() ? nullptr
2412 : &IPLS->DefGeneratorNonCandidates);
2413 LLVM_DEBUG({
2414 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2415 << "\n";
2416 });
2417
2418 // If this lookup was resumed after auto-suspension but all candidates
2419 // have already been generated (by some previous call to the generator)
2420 // treat the lookup as if it had completed generation.
2421 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2422 IPLS->DefGeneratorCandidates.empty())
2423 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2424 });
2425
2426 // If we encountered an error while filtering generation candidates then
2427 // bail out.
2428 if (Err)
2429 return IPLS->fail(Err: std::move(Err));
2430
2431 /// Apply any definition generators on the stack.
2432 LLVM_DEBUG({
2433 if (IPLS->CurDefGeneratorStack.empty())
2434 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2435 else if (IPLS->DefGeneratorCandidates.empty())
2436 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2437 else
2438 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2439 << " remaining generators for "
2440 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2441 });
2442 while (!IPLS->CurDefGeneratorStack.empty() &&
2443 !IPLS->DefGeneratorCandidates.empty()) {
2444 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2445
2446 if (!DG)
2447 return IPLS->fail(Err: make_error<StringError>(
2448 Args: "DefinitionGenerator removed while lookup in progress",
2449 Args: inconvertibleErrorCode()));
2450
2451 // At this point the lookup is in either the NotInGenerator state, or in
2452 // the ResumedForGenerator state.
2453 // If this lookup is in the NotInGenerator state then check whether the
2454 // generator is in use. If the generator is not in use then move the
2455 // lookup to the InGenerator state and continue. If the generator is
2456 // already in use then just add this lookup to the pending lookups list
2457 // and bail out.
2458 // If this lookup is in the ResumedForGenerator state then just move it
2459 // to InGenerator and continue.
2460 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2461 std::lock_guard<std::mutex> Lock(DG->M);
2462 if (DG->InUse) {
2463 DG->PendingLookups.push_back(x: std::move(IPLS));
2464 return;
2465 }
2466 DG->InUse = true;
2467 }
2468
2469 IPLS->GenState = InProgressLookupState::InGenerator;
2470
2471 auto K = IPLS->K;
2472 auto &LookupSet = IPLS->DefGeneratorCandidates;
2473
2474 // Run the generator. If the generator takes ownership of QA then this
2475 // will break the loop.
2476 {
2477 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2478 LookupState LS(std::move(IPLS));
2479 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2480 IPLS = std::move(LS.IPLS);
2481 }
2482
2483 // If the lookup returned then pop the generator stack and unblock the
2484 // next lookup on this generator (if any).
2485 if (IPLS)
2486 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2487
2488 // If there was an error then fail the query.
2489 if (Err) {
2490 LLVM_DEBUG({
2491 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2492 });
2493 assert(IPLS && "LS cannot be retained if error is returned");
2494 return IPLS->fail(Err: std::move(Err));
2495 }
2496
2497 // Otherwise if QA was captured then break the loop.
2498 if (!IPLS) {
2499 LLVM_DEBUG(
2500 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2501 return;
2502 }
2503
2504 // Otherwise if we're continuing around the loop then update candidates
2505 // for the next round.
2506 runSessionLocked(F: [&] {
2507 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2508 Err = IL_updateCandidatesFor(
2509 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2510 NonCandidates: JD.DefGenerators.empty() ? nullptr
2511 : &IPLS->DefGeneratorNonCandidates);
2512 });
2513
2514 // If updating candidates failed then fail the query.
2515 if (Err) {
2516 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2517 return IPLS->fail(Err: std::move(Err));
2518 }
2519 }
2520
2521 if (IPLS->DefGeneratorCandidates.empty() &&
2522 IPLS->DefGeneratorNonCandidates.empty()) {
2523 // Early out if there are no remaining symbols.
2524 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2525 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2526 break;
2527 } else {
2528 // If we get here then we've moved on to the next JITDylib with candidates
2529 // remaining.
2530 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2531 ++IPLS->CurSearchOrderIndex;
2532 IPLS->NewJITDylib = true;
2533 }
2534 }
2535
2536 // Remove any weakly referenced candidates that could not be found/generated.
2537 IPLS->DefGeneratorCandidates.remove_if(
2538 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2539 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2540 });
2541
2542 // If we get here then we've finished searching all JITDylibs.
2543 // If we matched all symbols then move to phase 2, otherwise fail the query
2544 // with a SymbolsNotFound error.
2545 if (IPLS->DefGeneratorCandidates.empty()) {
2546 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2547 IPLS->complete(IPLS: std::move(IPLS));
2548 } else {
2549 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2550 IPLS->fail(Err: make_error<SymbolsNotFound>(
2551 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2552 }
2553}
2554
2555void ExecutionSession::OL_completeLookup(
2556 std::unique_ptr<InProgressLookupState> IPLS,
2557 std::shared_ptr<AsynchronousSymbolQuery> Q,
2558 RegisterDependenciesFunction RegisterDependencies) {
2559
2560 LLVM_DEBUG({
2561 dbgs() << "Entering OL_completeLookup:\n"
2562 << " Lookup kind: " << IPLS->K << "\n"
2563 << " Search order: " << IPLS->SearchOrder
2564 << ", Current index = " << IPLS->CurSearchOrderIndex
2565 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2566 << " Lookup set: " << IPLS->LookupSet << "\n"
2567 << " Definition generator candidates: "
2568 << IPLS->DefGeneratorCandidates << "\n"
2569 << " Definition generator non-candidates: "
2570 << IPLS->DefGeneratorNonCandidates << "\n";
2571 });
2572
2573 bool QueryComplete = false;
2574 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2575
2576 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2577 for (auto &KV : IPLS->SearchOrder) {
2578 auto &JD = *KV.first;
2579 auto JDLookupFlags = KV.second;
2580 LLVM_DEBUG({
2581 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2582 << ") with lookup set " << IPLS->LookupSet << ":\n";
2583 });
2584
2585 auto Err = IPLS->LookupSet.forEachWithRemoval(
2586 Body: [&](const SymbolStringPtr &Name,
2587 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2588 LLVM_DEBUG({
2589 dbgs() << " Attempting to match \"" << Name << "\" ("
2590 << SymLookupFlags << ")... ";
2591 });
2592
2593 /// Search for the symbol. If not found then continue without
2594 /// removal.
2595 auto SymI = JD.Symbols.find(Val: Name);
2596 if (SymI == JD.Symbols.end()) {
2597 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2598 return false;
2599 }
2600
2601 // If this is a non-exported symbol and we're matching exported
2602 // symbols only then skip this symbol without removal.
2603 if (!SymI->second.getFlags().isExported() &&
2604 JDLookupFlags ==
2605 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2606 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2607 return false;
2608 }
2609
2610 // If we match against a materialization-side-effects only symbol
2611 // then make sure it is weakly-referenced. Otherwise bail out with
2612 // an error.
2613 // FIXME: Use a "materialization-side-effects-only symbols must be
2614 // weakly referenced" specific error here to reduce confusion.
2615 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2616 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2617 LLVM_DEBUG({
2618 dbgs() << "error: "
2619 "required, but symbol is has-side-effects-only\n";
2620 });
2621 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2622 Args: SymbolNameVector({Name}));
2623 }
2624
2625 // If we matched against this symbol but it is in the error state
2626 // then bail out and treat it as a failure to materialize.
2627 if (SymI->second.getFlags().hasError()) {
2628 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2629 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2630 (*FailedSymbolsMap)[&JD] = {Name};
2631 return make_error<FailedToMaterialize>(
2632 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2633 }
2634
2635 // Otherwise this is a match.
2636
2637 // If this symbol is already in the required state then notify the
2638 // query, remove the symbol and continue.
2639 if (SymI->second.getState() >= Q->getRequiredState()) {
2640 LLVM_DEBUG(dbgs()
2641 << "matched, symbol already in required state\n");
2642 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2643 return true;
2644 }
2645
2646 // Otherwise this symbol does not yet meet the required state. Check
2647 // whether it has a materializer attached, and if so prepare to run
2648 // it.
2649 if (SymI->second.hasMaterializerAttached()) {
2650 assert(SymI->second.getAddress() == ExecutorAddr() &&
2651 "Symbol not resolved but already has address?");
2652 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2653 assert(UMII != JD.UnmaterializedInfos.end() &&
2654 "Lazy symbol should have UnmaterializedInfo");
2655
2656 auto UMI = UMII->second;
2657 assert(UMI->MU && "Materializer should not be null");
2658 assert(UMI->RT && "Tracker should not be null");
2659 LLVM_DEBUG({
2660 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2661 << " (" << UMI->MU->getName() << ")\n";
2662 });
2663
2664 // Move all symbols associated with this MaterializationUnit into
2665 // materializing state.
2666 for (auto &KV : UMI->MU->getSymbols()) {
2667 auto SymK = JD.Symbols.find(Val: KV.first);
2668 assert(SymK != JD.Symbols.end() &&
2669 "No entry for symbol covered by MaterializationUnit");
2670 SymK->second.setMaterializerAttached(false);
2671 SymK->second.setState(SymbolState::Materializing);
2672 JD.UnmaterializedInfos.erase(Val: KV.first);
2673 }
2674
2675 // Add MU to the list of MaterializationUnits to be materialized.
2676 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2677 } else
2678 LLVM_DEBUG(dbgs() << "matched, registering query");
2679
2680 // Add the query to the PendingQueries list and continue, deleting
2681 // the element from the lookup set.
2682 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2683 SymI->second.getState() != SymbolState::Ready &&
2684 "By this line the symbol should be materializing");
2685 auto &MI = JD.MaterializingInfos[Name];
2686 MI.addQuery(Q);
2687 Q->addQueryDependence(JD, Name);
2688
2689 return true;
2690 });
2691
2692 JD.shrinkMaterializationInfoMemory();
2693
2694 // Handle failure.
2695 if (Err) {
2696
2697 LLVM_DEBUG({
2698 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2699 });
2700
2701 // Detach the query.
2702 Q->detach();
2703
2704 // Replace the MUs.
2705 for (auto &KV : CollectedUMIs) {
2706 auto &JD = *KV.first;
2707 for (auto &UMI : KV.second)
2708 for (auto &KV2 : UMI->MU->getSymbols()) {
2709 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2710 "Unexpected materializer in map");
2711 auto SymI = JD.Symbols.find(Val: KV2.first);
2712 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2713 assert(SymI->second.getState() == SymbolState::Materializing &&
2714 "Can not replace symbol that is not materializing");
2715 assert(!SymI->second.hasMaterializerAttached() &&
2716 "MaterializerAttached flag should not be set");
2717 SymI->second.setMaterializerAttached(true);
2718 JD.UnmaterializedInfos[KV2.first] = UMI;
2719 }
2720 }
2721
2722 return Err;
2723 }
2724 }
2725
2726 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2727 IPLS->LookupSet.forEachWithRemoval(
2728 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2729 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2730 Q->dropSymbol(Name);
2731 return true;
2732 } else
2733 return false;
2734 });
2735
2736 if (!IPLS->LookupSet.empty()) {
2737 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2738 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2739 Args: IPLS->LookupSet.getSymbolNames());
2740 }
2741
2742 // Record whether the query completed.
2743 QueryComplete = Q->isComplete();
2744
2745 LLVM_DEBUG({
2746 dbgs() << "Query successfully "
2747 << (QueryComplete ? "completed" : "lodged") << "\n";
2748 });
2749
2750 // Move the collected MUs to the OutstandingMUs list.
2751 if (!CollectedUMIs.empty()) {
2752 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2753
2754 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2755 for (auto &KV : CollectedUMIs) {
2756 LLVM_DEBUG({
2757 auto &JD = *KV.first;
2758 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2759 << " MUs.\n";
2760 });
2761 for (auto &UMI : KV.second) {
2762 auto MR = createMaterializationResponsibility(
2763 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2764 InitSymbol: std::move(UMI->MU->InitSymbol));
2765 OutstandingMUs.push_back(
2766 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2767 }
2768 }
2769 } else
2770 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2771
2772 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2773 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2774 RegisterDependencies(Q->QueryRegistrations);
2775 } else
2776 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2777
2778 return Error::success();
2779 });
2780
2781 if (LodgingErr) {
2782 LLVM_DEBUG(dbgs() << "Failing query\n");
2783 Q->detach();
2784 Q->handleFailed(Err: std::move(LodgingErr));
2785 return;
2786 }
2787
2788 if (QueryComplete) {
2789 LLVM_DEBUG(dbgs() << "Completing query\n");
2790 Q->handleComplete(ES&: *this);
2791 }
2792
2793 dispatchOutstandingMUs();
2794}
2795
2796void ExecutionSession::OL_completeLookupFlags(
2797 std::unique_ptr<InProgressLookupState> IPLS,
2798 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2799
2800 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2801 LLVM_DEBUG({
2802 dbgs() << "Entering OL_completeLookupFlags:\n"
2803 << " Lookup kind: " << IPLS->K << "\n"
2804 << " Search order: " << IPLS->SearchOrder
2805 << ", Current index = " << IPLS->CurSearchOrderIndex
2806 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2807 << " Lookup set: " << IPLS->LookupSet << "\n"
2808 << " Definition generator candidates: "
2809 << IPLS->DefGeneratorCandidates << "\n"
2810 << " Definition generator non-candidates: "
2811 << IPLS->DefGeneratorNonCandidates << "\n";
2812 });
2813
2814 SymbolFlagsMap Result;
2815
2816 // Attempt to find flags for each symbol.
2817 for (auto &KV : IPLS->SearchOrder) {
2818 auto &JD = *KV.first;
2819 auto JDLookupFlags = KV.second;
2820 LLVM_DEBUG({
2821 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2822 << ") with lookup set " << IPLS->LookupSet << ":\n";
2823 });
2824
2825 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2826 SymbolLookupFlags SymLookupFlags) {
2827 LLVM_DEBUG({
2828 dbgs() << " Attempting to match \"" << Name << "\" ("
2829 << SymLookupFlags << ")... ";
2830 });
2831
2832 // Search for the symbol. If not found then continue without removing
2833 // from the lookup set.
2834 auto SymI = JD.Symbols.find(Val: Name);
2835 if (SymI == JD.Symbols.end()) {
2836 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2837 return false;
2838 }
2839
2840 // If this is a non-exported symbol then it doesn't match. Skip it.
2841 if (!SymI->second.getFlags().isExported() &&
2842 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2843 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2844 return false;
2845 }
2846
2847 LLVM_DEBUG({
2848 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2849 << "\n";
2850 });
2851 Result[Name] = SymI->second.getFlags();
2852 return true;
2853 });
2854 }
2855
2856 // Remove any weakly referenced symbols that haven't been resolved.
2857 IPLS->LookupSet.remove_if(
2858 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2859 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2860 });
2861
2862 if (!IPLS->LookupSet.empty()) {
2863 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2864 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2865 Args: IPLS->LookupSet.getSymbolNames());
2866 }
2867
2868 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2869 return Result;
2870 });
2871
2872 // Run the callback on the result.
2873 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2874 OnComplete(std::move(Result));
2875}
2876
2877void ExecutionSession::OL_destroyMaterializationResponsibility(
2878 MaterializationResponsibility &MR) {
2879
2880 assert(MR.SymbolFlags.empty() &&
2881 "All symbols should have been explicitly materialized or failed");
2882 MR.JD.unlinkMaterializationResponsibility(MR);
2883}
2884
2885SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2886 const MaterializationResponsibility &MR) {
2887 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2888}
2889
2890Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2891 const SymbolMap &Symbols) {
2892 LLVM_DEBUG({
2893 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2894 });
2895#ifndef NDEBUG
2896 for (auto &KV : Symbols) {
2897 auto I = MR.SymbolFlags.find(KV.first);
2898 assert(I != MR.SymbolFlags.end() &&
2899 "Resolving symbol outside this responsibility set");
2900 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2901 "Can't resolve materialization-side-effects-only symbol");
2902 assert((KV.second.getFlags() & ~JITSymbolFlags::Common) ==
2903 (I->second & ~JITSymbolFlags::Common) &&
2904 "Resolving symbol with incorrect flags");
2905 }
2906#endif
2907
2908 return MR.JD.resolve(MR, Resolved: Symbols);
2909}
2910
2911template <typename HandleNewDepFn>
2912void ExecutionSession::propagateExtraEmitDeps(
2913 std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
2914 HandleNewDepFn HandleNewDep) {
2915
2916 // Iterate to a fixed-point to propagate extra-emit dependencies through the
2917 // EDU graph.
2918 while (!Worklist.empty()) {
2919 auto &EDU = *Worklist.front();
2920 Worklist.pop_front();
2921
2922 assert(EDUInfos.count(&EDU) && "No info entry for EDU");
2923 auto &EDUInfo = EDUInfos[&EDU];
2924
2925 // Propagate new dependencies to users.
2926 for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
2927
2928 // UserEDUInfo only present if UserEDU has its own users.
2929 JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
2930 {
2931 auto UserEDUInfoItr = EDUInfos.find(Val: UserEDU);
2932 if (UserEDUInfoItr != EDUInfos.end())
2933 UserEDUInfo = &UserEDUInfoItr->second;
2934 }
2935
2936 for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
2937 auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
2938 DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
2939 for (auto Dep : Deps) {
2940 if (UserEDUDepsForJD.insert(V: Dep).second) {
2941 HandleNewDep(*UserEDU, *DepJD, Dep);
2942 if (UserEDUInfo) {
2943 if (!UserEDUNewDepsForJD) {
2944 // If UserEDU has no new deps then it's not in the worklist
2945 // yet, so add it.
2946 if (UserEDUInfo->NewDeps.empty())
2947 Worklist.push_back(x: UserEDU);
2948 UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
2949 }
2950 // Add (DepJD, Dep) to NewDeps.
2951 UserEDUNewDepsForJD->insert(V: Dep);
2952 }
2953 }
2954 }
2955 }
2956 }
2957
2958 EDUInfo.NewDeps.clear();
2959 }
2960}
2961
2962// Note: This method modifies the emitted set.
2963ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
2964 MaterializationResponsibility &MR,
2965 ArrayRef<SymbolDependenceGroup> EmittedDeps) {
2966
2967 auto &TargetJD = MR.getTargetJITDylib();
2968
2969 // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
2970 // Symbol -> EmissionDepUnit mappings.
2971 DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
2972 EDUInfos.reserve(NumEntries: EmittedDeps.size());
2973 DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
2974 for (auto &DG : EmittedDeps) {
2975 assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
2976
2977 // Skip empty EDUs.
2978 if (DG.Dependencies.empty())
2979 continue;
2980
2981 auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(args&: TargetJD);
2982 auto &EDUInfo = EDUInfos[TmpEDU.get()];
2983 EDUInfo.EDU = std::move(TmpEDU);
2984 for (const auto &Symbol : DG.Symbols) {
2985 NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
2986 assert(!EDUForSymbol.count(NonOwningSymbol) &&
2987 "Symbol should not appear in more than one SymbolDependenceGroup");
2988 assert(MR.getSymbols().count(Symbol) &&
2989 "Symbol in DepGroups not in the emitted set");
2990 auto NewlyEmittedItr = MR.getSymbols().find(Val: Symbol);
2991 EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
2992 EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
2993 }
2994 }
2995
2996 // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
2997 {
2998 DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
2999 for (auto &[Sym, Flags] : MR.getSymbols()) {
3000 if (!EDUForSymbol.count(Val: NonOwningSymbolStringPtr(Sym)))
3001 ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
3002 }
3003 if (!ResidualSymbolFlags.empty()) {
3004 auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(args&: TargetJD);
3005 ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
3006 auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
3007 ResidualEDUInfo.EDU = std::move(ResidualEDU);
3008
3009 // If the residual EDU is the only one then bail out early.
3010 if (EDUInfos.size() == 1)
3011 return EDUInfos;
3012
3013 // Otherwise add the residual EDU to the EDUForSymbol map.
3014 for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
3015 EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
3016 }
3017 }
3018
3019#ifndef NDEBUG
3020 assert(EDUForSymbol.size() == MR.getSymbols().size() &&
3021 "MR symbols not fully covered by EDUs?");
3022 for (auto &[Sym, Flags] : MR.getSymbols()) {
3023 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
3024 "Sym in MR not covered by EDU");
3025 }
3026#endif // NDEBUG
3027
3028 // 3. Use the DepGroups array to build a graph of dependencies between
3029 // EmissionDepUnits in this finalization. We want to remove these
3030 // intra-finalization uses, propagating dependencies on symbols outside
3031 // this finalization. Add EDUs to the worklist.
3032 for (auto &DG : EmittedDeps) {
3033
3034 // Skip SymbolDependenceGroups with no dependencies.
3035 if (DG.Dependencies.empty())
3036 continue;
3037
3038 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
3039 "No EDU for DG");
3040 auto &EDU =
3041 *EDUForSymbol.find(Val: NonOwningSymbolStringPtr(*DG.Symbols.begin()))
3042 ->second;
3043
3044 for (auto &[DepJD, Deps] : DG.Dependencies) {
3045 DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
3046
3047 assert(!Deps.empty() && "Dependence set for DepJD is empty");
3048
3049 if (DepJD != &TargetJD) {
3050 // DepJD is some other JITDylib.There can't be any intra-finalization
3051 // edges here, so just skip.
3052 for (auto &Dep : Deps)
3053 NewDepsForJD.insert(V: NonOwningSymbolStringPtr(Dep));
3054 } else {
3055 // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
3056 // skipping any and recording the intra-finalization use instead.
3057 for (auto &Dep : Deps) {
3058 NonOwningSymbolStringPtr NonOwningDep(Dep);
3059 auto I = EDUForSymbol.find(Val: NonOwningDep);
3060 if (I == EDUForSymbol.end()) {
3061 if (!MR.getSymbols().count(Val: Dep))
3062 NewDepsForJD.insert(V: NonOwningDep);
3063 continue;
3064 }
3065
3066 if (I->second != &EDU)
3067 EDUInfos[I->second].IntraEmitUsers.insert(V: &EDU);
3068 }
3069 }
3070
3071 if (!NewDepsForJD.empty())
3072 EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
3073 }
3074 }
3075
3076 // 4. Build the worklist.
3077 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3078 for (auto &[EDU, EDUInfo] : EDUInfos) {
3079 // If this EDU has extra-finalization dependencies and intra-finalization
3080 // users then add it to the worklist.
3081 if (!EDU->Dependencies.empty()) {
3082 auto I = EDUInfos.find(Val: EDU);
3083 if (I != EDUInfos.end()) {
3084 auto &EDUInfo = I->second;
3085 if (!EDUInfo.IntraEmitUsers.empty()) {
3086 EDUInfo.NewDeps = EDU->Dependencies;
3087 Worklist.push_back(x: EDU);
3088 }
3089 }
3090 }
3091 }
3092
3093 // 4. Propagate dependencies through the EDU graph.
3094 propagateExtraEmitDeps(
3095 Worklist, EDUInfos,
3096 HandleNewDep: [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
3097
3098 return EDUInfos;
3099}
3100
3101void ExecutionSession::IL_makeEDUReady(
3102 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3103 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3104
3105 // The symbols for this EDU are ready.
3106 auto &JD = *EDU->JD;
3107
3108 for (auto &[Sym, Flags] : EDU->Symbols) {
3109 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3110 "JD does not have an entry for Sym");
3111 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3112
3113 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3114 Entry.getState() == SymbolState::Materializing) ||
3115 Entry.getState() == SymbolState::Resolved ||
3116 Entry.getState() == SymbolState::Emitted) &&
3117 "Emitting from state other than Resolved");
3118
3119 Entry.setState(SymbolState::Ready);
3120
3121 auto MII = JD.MaterializingInfos.find(Val: SymbolStringPtr(Sym));
3122
3123 // Check for pending queries.
3124 if (MII == JD.MaterializingInfos.end())
3125 continue;
3126 auto &MI = MII->second;
3127
3128 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Ready)) {
3129 Q->notifySymbolMetRequiredState(Name: SymbolStringPtr(Sym), Sym: Entry.getSymbol());
3130 if (Q->isComplete())
3131 Queries.insert(x: Q);
3132 Q->removeQueryDependence(JD, Name: SymbolStringPtr(Sym));
3133 }
3134
3135 JD.MaterializingInfos.erase(I: MII);
3136 }
3137
3138 JD.shrinkMaterializationInfoMemory();
3139}
3140
3141void ExecutionSession::IL_makeEDUEmitted(
3142 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3143 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3144
3145 // The symbols for this EDU are emitted, but not ready.
3146 auto &JD = *EDU->JD;
3147
3148 for (auto &[Sym, Flags] : EDU->Symbols) {
3149 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3150 "JD does not have an entry for Sym");
3151 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3152
3153 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3154 Entry.getState() == SymbolState::Materializing) ||
3155 Entry.getState() == SymbolState::Resolved ||
3156 Entry.getState() == SymbolState::Emitted) &&
3157 "Emitting from state other than Resolved");
3158
3159 if (Entry.getState() == SymbolState::Emitted) {
3160 // This was already emitted, so we can skip the rest of this loop.
3161#ifndef NDEBUG
3162 for (auto &[Sym, Flags] : EDU->Symbols) {
3163 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3164 "JD does not have an entry for Sym");
3165 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3166 assert(Entry.getState() == SymbolState::Emitted &&
3167 "Symbols for EDU in inconsistent state");
3168 assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
3169 "Emitted symbol has no MI");
3170 auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3171 assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
3172 "Already-emitted symbol has waiting-on-emitted queries");
3173 }
3174#endif // NDEBUG
3175 break;
3176 }
3177
3178 Entry.setState(SymbolState::Emitted);
3179 auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3180 MI.DefiningEDU = EDU;
3181
3182 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Emitted)) {
3183 Q->notifySymbolMetRequiredState(Name: SymbolStringPtr(Sym), Sym: Entry.getSymbol());
3184 if (Q->isComplete())
3185 Queries.insert(x: Q);
3186 Q->removeQueryDependence(JD, Name: SymbolStringPtr(Sym));
3187 }
3188 }
3189
3190 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3191 for (auto &Dep : Deps)
3192 DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
3193 V: EDU.get());
3194 }
3195}
3196
3197/// Removes the given dependence from EDU. If EDU's dependence set becomes
3198/// empty then this function adds an entry for it to the EDUInfos map.
3199/// Returns true if a new EDUInfosMap entry is added.
3200bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
3201 JITDylib &DepJD,
3202 NonOwningSymbolStringPtr DepSym,
3203 EDUInfosMap &EDUInfos) {
3204 assert(EDU.Dependencies.count(&DepJD) &&
3205 "JD does not appear in Dependencies of DependantEDU");
3206 assert(EDU.Dependencies[&DepJD].count(DepSym) &&
3207 "Symbol does not appear in Dependencies of DependantEDU");
3208 auto &JDDeps = EDU.Dependencies[&DepJD];
3209 JDDeps.erase(V: DepSym);
3210 if (JDDeps.empty()) {
3211 EDU.Dependencies.erase(Val: &DepJD);
3212 if (EDU.Dependencies.empty()) {
3213 // If the dependencies set has become empty then EDU _may_ be ready
3214 // (we won't know for sure until we've propagated the extra-emit deps).
3215 // Create an EDUInfo for it (if it doesn't have one already) so that
3216 // it'll be visited after propagation.
3217 auto &DepEDUInfo = EDUInfos[&EDU];
3218 if (!DepEDUInfo.EDU) {
3219 assert(EDU.JD->Symbols.count(
3220 SymbolStringPtr(EDU.Symbols.begin()->first)) &&
3221 "Missing symbol entry for first symbol in EDU");
3222 auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
3223 Val: SymbolStringPtr(EDU.Symbols.begin()->first));
3224 assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
3225 "Missing MI for first symbol in DependantEDU");
3226 DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
3227 return true;
3228 }
3229 }
3230 }
3231 return false;
3232}
3233
3234Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
3235 JITDylib &ClosedJD) {
3236 SymbolNameSet FailedSymbols;
3237 for (auto &[Sym, Flags] : EDU.Symbols)
3238 FailedSymbols.insert(V: SymbolStringPtr(Sym));
3239 SymbolDependenceMap BadDeps;
3240 for (auto &Dep : EDU.Dependencies[&ClosedJD])
3241 BadDeps[&ClosedJD].insert(V: SymbolStringPtr(Dep));
3242 return make_error<UnsatisfiedSymbolDependencies>(
3243 Args: ClosedJD.getExecutionSession().getSymbolStringPool(), Args&: EDU.JD,
3244 Args: std::move(FailedSymbols), Args: std::move(BadDeps),
3245 Args: ClosedJD.getName() + " is closed");
3246}
3247
3248Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
3249 JITDylib &BadJD,
3250 SymbolNameSet BadDeps) {
3251 SymbolNameSet FailedSymbols;
3252 for (auto &[Sym, Flags] : EDU.Symbols)
3253 FailedSymbols.insert(V: SymbolStringPtr(Sym));
3254 SymbolDependenceMap BadDepsMap;
3255 BadDepsMap[&BadJD] = std::move(BadDeps);
3256 return make_error<UnsatisfiedSymbolDependencies>(
3257 Args: BadJD.getExecutionSession().getSymbolStringPool(), Args: &BadJD,
3258 Args: std::move(FailedSymbols), Args: std::move(BadDepsMap),
3259 Args: "dependencies removed or in error state");
3260}
3261
3262Expected<JITDylib::AsynchronousSymbolQuerySet>
3263ExecutionSession::IL_emit(MaterializationResponsibility &MR,
3264 EDUInfosMap EDUInfos) {
3265
3266 if (MR.RT->isDefunct())
3267 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
3268
3269 auto &TargetJD = MR.getTargetJITDylib();
3270 if (TargetJD.State != JITDylib::Open)
3271 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
3272 " is defunct",
3273 Args: inconvertibleErrorCode());
3274#ifdef EXPENSIVE_CHECKS
3275 verifySessionState("entering ExecutionSession::IL_emit");
3276#endif
3277
3278 // Walk all EDUs:
3279 // 1. Verifying that dependencies are available (not removed or in the error
3280 // state.
3281 // 2. Removing any dependencies that are already Ready.
3282 // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
3283 // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
3284 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3285 for (auto &[EDU, _] : EDUInfos)
3286 Worklist.push_back(x: EDU);
3287
3288 for (auto *EDU : Worklist) {
3289 auto *EDUInfo = &EDUInfos[EDU];
3290
3291 SmallVector<JITDylib *> DepJDsToRemove;
3292 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3293 if (DepJD->State != JITDylib::Open)
3294 return makeJDClosedError(EDU&: *EDU, ClosedJD&: *DepJD);
3295
3296 SymbolNameSet BadDeps;
3297 SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
3298 for (auto &Dep : Deps) {
3299 auto DepEntryItr = DepJD->Symbols.find(Val: SymbolStringPtr(Dep));
3300
3301 // If this dep has been removed or moved to the error state then add it
3302 // to the bad deps set. We aggregate these bad deps for more
3303 // comprehensive error messages.
3304 if (DepEntryItr == DepJD->Symbols.end() ||
3305 DepEntryItr->second.getFlags().hasError()) {
3306 BadDeps.insert(V: SymbolStringPtr(Dep));
3307 continue;
3308 }
3309
3310 // If this dep isn't emitted yet then just add it to the NewDeps set to
3311 // be propagated.
3312 auto &DepEntry = DepEntryItr->second;
3313 if (DepEntry.getState() < SymbolState::Emitted) {
3314 EDUInfo->NewDeps[DepJD].insert(V: Dep);
3315 continue;
3316 }
3317
3318 // This dep has been emitted, so add it to the list to be removed from
3319 // EDU.
3320 DepsToRemove.push_back(Elt: Dep);
3321
3322 // If Dep is Ready then there's nothing further to do.
3323 if (DepEntry.getState() == SymbolState::Ready) {
3324 assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3325 "Unexpected MaterializationInfo attached to ready symbol");
3326 continue;
3327 }
3328
3329 // If we get here thene Dep is Emitted. We need to look up its defining
3330 // EDU and add this EDU to the defining EDU's list of users (this means
3331 // creating an EDUInfos entry if the defining EDU doesn't have one
3332 // already).
3333 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3334 "Expected MaterializationInfo for emitted dependency");
3335 auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
3336 assert(DepMI.DefiningEDU &&
3337 "Emitted symbol does not have a defining EDU");
3338 assert(!DepMI.DefiningEDU->Dependencies.empty() &&
3339 "Emitted symbol has empty dependencies (should be ready)");
3340 assert(DepMI.DependantEDUs.empty() &&
3341 "Already-emitted symbol has dependant EDUs?");
3342 auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
3343 if (!DepEDUInfo.EDU) {
3344 // No EDUInfo yet -- build initial entry, and reset the EDUInfo
3345 // pointer, which we will have invalidated.
3346 EDUInfo = &EDUInfos[EDU];
3347 DepEDUInfo.EDU = DepMI.DefiningEDU;
3348 for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
3349 if (DepDepJD == &TargetJD) {
3350 for (auto &DepDep : DepDeps)
3351 if (!MR.getSymbols().count(Val: SymbolStringPtr(DepDep)))
3352 DepEDUInfo.NewDeps[DepDepJD].insert(V: DepDep);
3353 } else
3354 DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
3355 }
3356 }
3357 DepEDUInfo.IntraEmitUsers.insert(V: EDU);
3358 }
3359
3360 // Some dependencies were removed or in an error state -- error out.
3361 if (!BadDeps.empty())
3362 return makeUnsatisfiedDepsError(EDU&: *EDU, BadJD&: *DepJD, BadDeps: std::move(BadDeps));
3363
3364 // Remove the emitted / ready deps from DepJD.
3365 for (auto &Dep : DepsToRemove)
3366 Deps.erase(V: Dep);
3367
3368 // If there are no further deps in DepJD then flag it for removal too.
3369 if (Deps.empty())
3370 DepJDsToRemove.push_back(Elt: DepJD);
3371 }
3372
3373 // Remove any JDs whose dependence sets have become empty.
3374 for (auto &DepJD : DepJDsToRemove) {
3375 assert(EDU->Dependencies.count(DepJD) &&
3376 "Trying to remove non-existent dep entries");
3377 EDU->Dependencies.erase(Val: DepJD);
3378 }
3379
3380 // Now look for users of this EDU.
3381 for (auto &[Sym, Flags] : EDU->Symbols) {
3382 assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
3383 "Sym not present in symbol table");
3384 assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
3385 SymbolState::Resolved ||
3386 TargetJD.Symbols[SymbolStringPtr(Sym)]
3387 .getFlags()
3388 .hasMaterializationSideEffectsOnly()) &&
3389 "Emitting symbol not in the resolved state");
3390 assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
3391 "Symbol is already in an error state");
3392
3393 auto MII = TargetJD.MaterializingInfos.find(Val: SymbolStringPtr(Sym));
3394 if (MII == TargetJD.MaterializingInfos.end() ||
3395 MII->second.DependantEDUs.empty())
3396 continue;
3397
3398 for (auto &DependantEDU : MII->second.DependantEDUs) {
3399 if (IL_removeEDUDependence(EDU&: *DependantEDU, DepJD&: TargetJD, DepSym: Sym, EDUInfos))
3400 EDUInfo = &EDUInfos[EDU];
3401 EDUInfo->IntraEmitUsers.insert(V: DependantEDU);
3402 }
3403 MII->second.DependantEDUs.clear();
3404 }
3405 }
3406
3407 Worklist.clear();
3408 for (auto &[EDU, EDUInfo] : EDUInfos) {
3409 if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
3410 if (EDUInfo.NewDeps.empty())
3411 EDUInfo.NewDeps = EDU->Dependencies;
3412 Worklist.push_back(x: EDU);
3413 }
3414 }
3415
3416 propagateExtraEmitDeps(
3417 Worklist, EDUInfos,
3418 HandleNewDep: [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
3419 NonOwningSymbolStringPtr Sym) {
3420 JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(V: &EDU);
3421 });
3422
3423 JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
3424
3425 // Extract completed queries and lodge not-yet-ready EDUs in the
3426 // session.
3427 for (auto &[EDU, EDUInfo] : EDUInfos) {
3428 if (EDU->Dependencies.empty())
3429 IL_makeEDUReady(EDU: std::move(EDUInfo.EDU), Queries&: CompletedQueries);
3430 else
3431 IL_makeEDUEmitted(EDU: std::move(EDUInfo.EDU), Queries&: CompletedQueries);
3432 }
3433
3434#ifdef EXPENSIVE_CHECKS
3435 verifySessionState("exiting ExecutionSession::IL_emit");
3436#endif
3437
3438 return std::move(CompletedQueries);
3439}
3440
3441Error ExecutionSession::OL_notifyEmitted(
3442 MaterializationResponsibility &MR,
3443 ArrayRef<SymbolDependenceGroup> DepGroups) {
3444 LLVM_DEBUG({
3445 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
3446 << "\n";
3447 if (!DepGroups.empty()) {
3448 dbgs() << " Initial dependencies:\n";
3449 for (auto &SDG : DepGroups) {
3450 dbgs() << " Symbols: " << SDG.Symbols
3451 << ", Dependencies: " << SDG.Dependencies << "\n";
3452 }
3453 }
3454 });
3455
3456#ifndef NDEBUG
3457 SymbolNameSet Visited;
3458 for (auto &DG : DepGroups) {
3459 for (auto &Sym : DG.Symbols) {
3460 assert(MR.SymbolFlags.count(Sym) &&
3461 "DG contains dependence for symbol outside this MR");
3462 assert(Visited.insert(Sym).second &&
3463 "DG contains duplicate entries for Name");
3464 }
3465 }
3466#endif // NDEBUG
3467
3468 auto EDUInfos = simplifyDepGroups(MR, EmittedDeps: DepGroups);
3469
3470 LLVM_DEBUG({
3471 dbgs() << " Simplified dependencies:\n";
3472 for (auto &[EDU, EDUInfo] : EDUInfos) {
3473 dbgs() << " Symbols: { ";
3474 for (auto &[Sym, Flags] : EDU->Symbols)
3475 dbgs() << Sym << " ";
3476 dbgs() << "}, Dependencies: { ";
3477 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3478 dbgs() << "(" << DepJD->getName() << ", { ";
3479 for (auto &Dep : Deps)
3480 dbgs() << Dep << " ";
3481 dbgs() << "}) ";
3482 }
3483 dbgs() << "}\n";
3484 }
3485 });
3486
3487 auto CompletedQueries =
3488 runSessionLocked(F: [&]() { return IL_emit(MR, EDUInfos); });
3489
3490 // On error bail out.
3491 if (!CompletedQueries)
3492 return CompletedQueries.takeError();
3493
3494 MR.SymbolFlags.clear();
3495
3496 // Otherwise notify all the completed queries.
3497 for (auto &Q : *CompletedQueries) {
3498 assert(Q->isComplete() && "Q is not complete");
3499 Q->handleComplete(ES&: *this);
3500 }
3501
3502 return Error::success();
3503}
3504
3505Error ExecutionSession::OL_defineMaterializing(
3506 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3507
3508 LLVM_DEBUG({
3509 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3510 << NewSymbolFlags << "\n";
3511 });
3512 if (auto AcceptedDefs =
3513 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3514 // Add all newly accepted symbols to this responsibility object.
3515 for (auto &KV : *AcceptedDefs)
3516 MR.SymbolFlags.insert(KV);
3517 return Error::success();
3518 } else
3519 return AcceptedDefs.takeError();
3520}
3521
3522std::pair<JITDylib::AsynchronousSymbolQuerySet,
3523 std::shared_ptr<SymbolDependenceMap>>
3524ExecutionSession::IL_failSymbols(JITDylib &JD,
3525 const SymbolNameVector &SymbolsToFail) {
3526
3527#ifdef EXPENSIVE_CHECKS
3528 verifySessionState("entering ExecutionSession::IL_failSymbols");
3529#endif
3530
3531 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3532 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3533 auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
3534 JITDylib::AsynchronousSymbolQueryList ToDetach;
3535 for (auto &Q : MI.pendingQueries()) {
3536 // Add the query to the list to be failed and detach it.
3537 FailedQueries.insert(x: Q);
3538 ToDetach.push_back(x: Q);
3539 }
3540 for (auto &Q : ToDetach)
3541 Q->detach();
3542 assert(!MI.hasQueriesPending() && "Queries still pending after detach");
3543 };
3544
3545 for (auto &Name : SymbolsToFail) {
3546 (*FailedSymbolsMap)[&JD].insert(V: Name);
3547
3548 // Look up the symbol to fail.
3549 auto SymI = JD.Symbols.find(Val: Name);
3550
3551 // FIXME: Revisit this. We should be able to assert sequencing between
3552 // ResourceTracker removal and symbol failure.
3553 //
3554 // It's possible that this symbol has already been removed, e.g. if a
3555 // materialization failure happens concurrently with a ResourceTracker or
3556 // JITDylib removal. In that case we can safely skip this symbol and
3557 // continue.
3558 if (SymI == JD.Symbols.end())
3559 continue;
3560 auto &Sym = SymI->second;
3561
3562 // If the symbol is already in the error state then we must have visited
3563 // it earlier.
3564 if (Sym.getFlags().hasError()) {
3565 assert(!JD.MaterializingInfos.count(Name) &&
3566 "Symbol in error state still has MaterializingInfo");
3567 continue;
3568 }
3569
3570 // Move the symbol into the error state.
3571 Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
3572
3573 // FIXME: Come up with a sane mapping of state to
3574 // presence-of-MaterializingInfo so that we can assert presence / absence
3575 // here, rather than testing it.
3576 auto MII = JD.MaterializingInfos.find(Val: Name);
3577 if (MII == JD.MaterializingInfos.end())
3578 continue;
3579
3580 auto &MI = MII->second;
3581
3582 // Collect queries to be failed for this MII.
3583 ExtractFailedQueries(MI);
3584
3585 if (MI.DefiningEDU) {
3586 // If there is a DefiningEDU for this symbol then remove this
3587 // symbol from it.
3588 assert(MI.DependantEDUs.empty() &&
3589 "Symbol with DefiningEDU should not have DependantEDUs");
3590 assert(Sym.getState() >= SymbolState::Emitted &&
3591 "Symbol has EDU, should have been emitted");
3592 assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
3593 "Symbol does not appear in its DefiningEDU");
3594 MI.DefiningEDU->Symbols.erase(Val: NonOwningSymbolStringPtr(Name));
3595 MI.DefiningEDU = nullptr;
3596 } else {
3597 // Otherwise if there are any EDUs waiting on this symbol then move
3598 // those symbols to the error state too, and deregister them from the
3599 // symbols that they depend on.
3600 // Note: We use a copy of DependantEDUs here since we'll be removing
3601 // from the original set as we go.
3602 for (auto &DependantEDU : MI.DependantEDUs) {
3603
3604 // Remove DependantEDU from all of its users DependantEDUs lists.
3605 for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
3606 for (auto DepSym : DepSyms) {
3607 // Skip self-reference to avoid invalidating the MI.DependantEDUs
3608 // map. We'll clear this later.
3609 if (DepJD == &JD && DepSym == Name)
3610 continue;
3611 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3612 "DepSym not in DepJD?");
3613 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3614 "DependantEDU not registered with symbol it depends on");
3615 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3616 assert(SymMI.DependantEDUs.count(DependantEDU) &&
3617 "DependantEDU missing from DependantEDUs list");
3618 SymMI.DependantEDUs.erase(V: DependantEDU);
3619 }
3620 }
3621
3622 // Move any symbols defined by DependantEDU into the error state and
3623 // fail any queries waiting on them.
3624 auto &DepJD = *DependantEDU->JD;
3625 auto DepEDUSymbols = std::move(DependantEDU->Symbols);
3626 for (auto &[DepName, Flags] : DepEDUSymbols) {
3627 auto DepSymItr = DepJD.Symbols.find(Val: SymbolStringPtr(DepName));
3628 assert(DepSymItr != DepJD.Symbols.end() &&
3629 "Symbol not present in table");
3630 auto &DepSym = DepSymItr->second;
3631
3632 assert(DepSym.getState() >= SymbolState::Emitted &&
3633 "Symbol has EDU, should have been emitted");
3634 assert(!DepSym.getFlags().hasError() &&
3635 "Symbol is already in the error state?");
3636 DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
3637 (*FailedSymbolsMap)[&DepJD].insert(V: SymbolStringPtr(DepName));
3638
3639 // This symbol has a defining EDU so its MaterializingInfo object must
3640 // exist.
3641 auto DepMIItr =
3642 DepJD.MaterializingInfos.find(Val: SymbolStringPtr(DepName));
3643 assert(DepMIItr != DepJD.MaterializingInfos.end() &&
3644 "Symbol has defining EDU but not MaterializingInfo");
3645 auto &DepMI = DepMIItr->second;
3646 assert(DepMI.DefiningEDU.get() == DependantEDU &&
3647 "Bad EDU dependence edge");
3648 assert(DepMI.DependantEDUs.empty() &&
3649 "Symbol was emitted, should not have any DependantEDUs");
3650 ExtractFailedQueries(DepMI);
3651 DepJD.MaterializingInfos.erase(Val: SymbolStringPtr(DepName));
3652 }
3653
3654 DepJD.shrinkMaterializationInfoMemory();
3655 }
3656
3657 MI.DependantEDUs.clear();
3658 }
3659
3660 assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
3661 assert(MI.DependantEDUs.empty() &&
3662 "DependantEDUs should have been removed above");
3663 assert(!MI.hasQueriesPending() &&
3664 "Can not delete MaterializingInfo with queries pending");
3665 JD.MaterializingInfos.erase(Val: Name);
3666 }
3667
3668 JD.shrinkMaterializationInfoMemory();
3669
3670#ifdef EXPENSIVE_CHECKS
3671 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3672#endif
3673
3674 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3675}
3676
3677void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3678
3679 LLVM_DEBUG({
3680 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3681 << MR.SymbolFlags << "\n";
3682 });
3683
3684 if (MR.SymbolFlags.empty())
3685 return;
3686
3687 SymbolNameVector SymbolsToFail;
3688 for (auto &[Name, Flags] : MR.SymbolFlags)
3689 SymbolsToFail.push_back(x: Name);
3690 MR.SymbolFlags.clear();
3691
3692 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3693 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3694
3695 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3696 // If the tracker is defunct then there's nothing to do here.
3697 if (MR.RT->isDefunct())
3698 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3699 std::shared_ptr<SymbolDependenceMap>>();
3700 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3701 });
3702
3703 for (auto &Q : FailedQueries)
3704 Q->handleFailed(
3705 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3706}
3707
3708Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3709 std::unique_ptr<MaterializationUnit> MU) {
3710 for (auto &KV : MU->getSymbols()) {
3711 assert(MR.SymbolFlags.count(KV.first) &&
3712 "Replacing definition outside this responsibility set");
3713 MR.SymbolFlags.erase(Val: KV.first);
3714 }
3715
3716 if (MU->getInitializerSymbol() == MR.InitSymbol)
3717 MR.InitSymbol = nullptr;
3718
3719 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3720 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3721 << "\n";
3722 }););
3723
3724 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3725}
3726
3727Expected<std::unique_ptr<MaterializationResponsibility>>
3728ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3729 const SymbolNameSet &Symbols) {
3730
3731 SymbolStringPtr DelegatedInitSymbol;
3732 SymbolFlagsMap DelegatedFlags;
3733
3734 for (auto &Name : Symbols) {
3735 auto I = MR.SymbolFlags.find(Val: Name);
3736 assert(I != MR.SymbolFlags.end() &&
3737 "Symbol is not tracked by this MaterializationResponsibility "
3738 "instance");
3739
3740 DelegatedFlags[Name] = std::move(I->second);
3741 if (Name == MR.InitSymbol)
3742 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3743
3744 MR.SymbolFlags.erase(I);
3745 }
3746
3747 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3748 InitSymbol: std::move(DelegatedInitSymbol));
3749}
3750
3751#ifndef NDEBUG
3752void ExecutionSession::dumpDispatchInfo(Task &T) {
3753 runSessionLocked([&]() {
3754 dbgs() << "Dispatching: ";
3755 T.printDescription(dbgs());
3756 dbgs() << "\n";
3757 });
3758}
3759#endif // NDEBUG
3760
3761} // End namespace orc.
3762} // End namespace llvm.
3763