1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
15#include "llvm/Support/FormatVariadic.h"
16#include "llvm/Support/MSVCErrorWorkarounds.h"
17#include "llvm/Support/raw_ostream.h"
18
19#include <condition_variable>
20#include <future>
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25namespace llvm {
26namespace orc {
27
28char ResourceTrackerDefunct::ID = 0;
29char JITDylibDefunct::ID = 0;
30char FailedToMaterialize::ID = 0;
31char SymbolsNotFound::ID = 0;
32char SymbolsCouldNotBeRemoved::ID = 0;
33char MissingSymbolDefinitions::ID = 0;
34char UnexpectedSymbolDefinitions::ID = 0;
35char UnsatisfiedSymbolDependencies::ID = 0;
36char MaterializationTask::ID = 0;
37char LookupTask::ID = 0;
38
39RegisterDependenciesFunction NoDependenciesToRegister =
40 RegisterDependenciesFunction();
41
42void MaterializationUnit::anchor() {}
43
44ResourceTracker::ResourceTracker(JITDylibSP JD) {
45 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
46 "JITDylib must be two byte aligned");
47 JD->Retain();
48 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
49}
50
51ResourceTracker::~ResourceTracker() {
52 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
53 getJITDylib().Release();
54}
55
56Error ResourceTracker::remove() {
57 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
58}
59
60void ResourceTracker::transferTo(ResourceTracker &DstRT) {
61 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
62}
63
64void ResourceTracker::makeDefunct() {
65 uintptr_t Val = JDAndFlag.load();
66 Val |= 0x1U;
67 JDAndFlag.store(i: Val);
68}
69
70ResourceManager::~ResourceManager() = default;
71
72ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
73 : RT(std::move(RT)) {}
74
75std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
76 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
77}
78
79void ResourceTrackerDefunct::log(raw_ostream &OS) const {
80 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
81}
82
83std::error_code JITDylibDefunct::convertToErrorCode() const {
84 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
85}
86
87void JITDylibDefunct::log(raw_ostream &OS) const {
88 OS << "JITDylib " << JD->getName() << " (" << (void *)JD.get()
89 << ") is defunct";
90}
91
92FailedToMaterialize::FailedToMaterialize(
93 std::shared_ptr<SymbolStringPool> SSP,
94 std::shared_ptr<SymbolDependenceMap> Symbols)
95 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
96 assert(this->SSP && "String pool cannot be null");
97 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
98
99 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
100 // don't have to manually retain/release.
101 for (auto &[JD, Syms] : *this->Symbols)
102 JD->Retain();
103}
104
105FailedToMaterialize::~FailedToMaterialize() {
106 for (auto &[JD, Syms] : *Symbols)
107 JD->Release();
108}
109
110std::error_code FailedToMaterialize::convertToErrorCode() const {
111 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
112}
113
114void FailedToMaterialize::log(raw_ostream &OS) const {
115 OS << "Failed to materialize symbols: " << *Symbols;
116}
117
118UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
119 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
120 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
121 std::string Explanation)
122 : SSP(std::move(SSP)), JD(std::move(JD)),
123 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
124 Explanation(std::move(Explanation)) {}
125
126std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
127 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
128}
129
130void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
131 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
132 << ", due to unsatisfied dependencies " << BadDeps;
133 if (!Explanation.empty())
134 OS << " (" << Explanation << ")";
135}
136
137SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
138 SymbolNameSet Symbols)
139 : SSP(std::move(SSP)) {
140 llvm::append_range(C&: this->Symbols, R&: Symbols);
141 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
142}
143
144SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
145 SymbolNameVector Symbols)
146 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
147 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
148}
149
150std::error_code SymbolsNotFound::convertToErrorCode() const {
151 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
152}
153
154void SymbolsNotFound::log(raw_ostream &OS) const {
155 OS << "Symbols not found: " << Symbols;
156}
157
158SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
159 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
160 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
161 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
162}
163
164std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
165 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
166}
167
168void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
169 OS << "Symbols could not be removed: " << Symbols;
170}
171
172std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
173 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
174}
175
176void MissingSymbolDefinitions::log(raw_ostream &OS) const {
177 OS << "Missing definitions in module " << ModuleName
178 << ": " << Symbols;
179}
180
181std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
182 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
183}
184
185void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
186 OS << "Unexpected definitions in module " << ModuleName
187 << ": " << Symbols;
188}
189
190void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
191 JD->getExecutionSession().lookup(
192 K: LookupKind::Static, SearchOrder: {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
193 Symbols: SymbolLookupSet(Name), RequiredState: SymbolState::Ready,
194 NotifyComplete: [OnComplete = std::move(OnComplete)
195#ifndef NDEBUG
196 ,
197 Name = this->Name // Captured for the assert below only.
198#endif // NDEBUG
199 ](Expected<SymbolMap> Result) mutable {
200 if (Result) {
201 assert(Result->size() == 1 && "Unexpected number of results");
202 assert(Result->count(Name) &&
203 "Result does not contain expected symbol");
204 OnComplete(Result->begin()->second);
205 } else
206 OnComplete(Result.takeError());
207 },
208 RegisterDependencies: NoDependenciesToRegister);
209}
210
211AsynchronousSymbolQuery::AsynchronousSymbolQuery(
212 const SymbolLookupSet &Symbols, SymbolState RequiredState,
213 SymbolsResolvedCallback NotifyComplete)
214 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
215 assert(RequiredState >= SymbolState::Resolved &&
216 "Cannot query for a symbols that have not reached the resolve state "
217 "yet");
218
219 OutstandingSymbolsCount = Symbols.size();
220
221 for (auto &[Name, Flags] : Symbols)
222 ResolvedSymbols[Name] = ExecutorSymbolDef();
223}
224
225void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
226 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
227 auto I = ResolvedSymbols.find(Val: Name);
228 assert(I != ResolvedSymbols.end() &&
229 "Resolving symbol outside the requested set");
230 assert(I->second == ExecutorSymbolDef() &&
231 "Redundantly resolving symbol Name");
232
233 // If this is a materialization-side-effects-only symbol then drop it,
234 // otherwise update its map entry with its resolved address.
235 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
236 ResolvedSymbols.erase(I);
237 else
238 I->second = std::move(Sym);
239 --OutstandingSymbolsCount;
240}
241
242void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
243 assert(OutstandingSymbolsCount == 0 &&
244 "Symbols remain, handleComplete called prematurely");
245
246 class RunQueryCompleteTask : public Task {
247 public:
248 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
249 SymbolsResolvedCallback NotifyComplete)
250 : ResolvedSymbols(std::move(ResolvedSymbols)),
251 NotifyComplete(std::move(NotifyComplete)) {}
252 void printDescription(raw_ostream &OS) override {
253 OS << "Execute query complete callback for " << ResolvedSymbols;
254 }
255 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
256
257 private:
258 SymbolMap ResolvedSymbols;
259 SymbolsResolvedCallback NotifyComplete;
260 };
261
262 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
263 args: std::move(NotifyComplete));
264 NotifyComplete = SymbolsResolvedCallback();
265 ES.dispatchTask(T: std::move(T));
266}
267
268void AsynchronousSymbolQuery::handleFailed(Error Err) {
269 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
270 OutstandingSymbolsCount == 0 &&
271 "Query should already have been abandoned");
272 NotifyComplete(std::move(Err));
273 NotifyComplete = SymbolsResolvedCallback();
274}
275
276void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
277 SymbolStringPtr Name) {
278 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
279 (void)Added;
280 assert(Added && "Duplicate dependence notification?");
281}
282
283void AsynchronousSymbolQuery::removeQueryDependence(
284 JITDylib &JD, const SymbolStringPtr &Name) {
285 auto QRI = QueryRegistrations.find(Val: &JD);
286 assert(QRI != QueryRegistrations.end() &&
287 "No dependencies registered for JD");
288 assert(QRI->second.count(Name) && "No dependency on Name in JD");
289 QRI->second.erase(V: Name);
290 if (QRI->second.empty())
291 QueryRegistrations.erase(I: QRI);
292}
293
294void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
295 auto I = ResolvedSymbols.find(Val: Name);
296 assert(I != ResolvedSymbols.end() &&
297 "Redundant removal of weakly-referenced symbol");
298 ResolvedSymbols.erase(I);
299 --OutstandingSymbolsCount;
300}
301
302void AsynchronousSymbolQuery::detach() {
303 ResolvedSymbols.clear();
304 OutstandingSymbolsCount = 0;
305 for (auto &[JD, Syms] : QueryRegistrations)
306 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
307 QueryRegistrations.clear();
308}
309
310ReExportsMaterializationUnit::ReExportsMaterializationUnit(
311 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
312 SymbolAliasMap Aliases)
313 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
314 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
315
316StringRef ReExportsMaterializationUnit::getName() const {
317 return "<Reexports>";
318}
319
320void ReExportsMaterializationUnit::materialize(
321 std::unique_ptr<MaterializationResponsibility> R) {
322
323 auto &ES = R->getTargetJITDylib().getExecutionSession();
324 JITDylib &TgtJD = R->getTargetJITDylib();
325 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
326
327 // Find the set of requested aliases and aliasees. Return any unrequested
328 // aliases back to the JITDylib so as to not prematurely materialize any
329 // aliasees.
330 auto RequestedSymbols = R->getRequestedSymbols();
331 SymbolAliasMap RequestedAliases;
332
333 for (auto &Name : RequestedSymbols) {
334 auto I = Aliases.find(Val: Name);
335 assert(I != Aliases.end() && "Symbol not found in aliases map?");
336 RequestedAliases[Name] = std::move(I->second);
337 Aliases.erase(I);
338 }
339
340 LLVM_DEBUG({
341 ES.runSessionLocked([&]() {
342 dbgs() << "materializing reexports: target = " << TgtJD.getName()
343 << ", source = " << SrcJD.getName() << " " << RequestedAliases
344 << "\n";
345 });
346 });
347
348 if (!Aliases.empty()) {
349 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
350 SourceJDLookupFlags))
351 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
352
353 if (Err) {
354 // FIXME: Should this be reported / treated as failure to materialize?
355 // Or should this be treated as a sanctioned bailing-out?
356 ES.reportError(Err: std::move(Err));
357 R->failMaterialization();
358 return;
359 }
360 }
361
362 // The OnResolveInfo struct will hold the aliases and responsibility for each
363 // query in the list.
364 struct OnResolveInfo {
365 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
366 SymbolAliasMap Aliases)
367 : R(std::move(R)), Aliases(std::move(Aliases)) {}
368
369 std::unique_ptr<MaterializationResponsibility> R;
370 SymbolAliasMap Aliases;
371 std::vector<SymbolDependenceGroup> SDGs;
372 };
373
374 // Build a list of queries to issue. In each round we build a query for the
375 // largest set of aliases that we can resolve without encountering a chain of
376 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
377 // query would be waiting on a symbol that it itself had to resolve. Creating
378 // a new query for each link in such a chain eliminates the possibility of
379 // deadlock. In practice chains are likely to be rare, and this algorithm will
380 // usually result in a single query to issue.
381
382 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
383 QueryInfos;
384 while (!RequestedAliases.empty()) {
385 SymbolNameSet ResponsibilitySymbols;
386 SymbolLookupSet QuerySymbols;
387 SymbolAliasMap QueryAliases;
388
389 // Collect as many aliases as we can without including a chain.
390 for (auto &[Alias, AliasInfo] : RequestedAliases) {
391 // Chain detected. Skip this symbol for this round.
392 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: AliasInfo.Aliasee) ||
393 RequestedAliases.count(Val: AliasInfo.Aliasee)))
394 continue;
395
396 ResponsibilitySymbols.insert(V: Alias);
397 QuerySymbols.add(Name: AliasInfo.Aliasee,
398 Flags: AliasInfo.AliasFlags.hasMaterializationSideEffectsOnly()
399 ? SymbolLookupFlags::WeaklyReferencedSymbol
400 : SymbolLookupFlags::RequiredSymbol);
401 QueryAliases[Alias] = std::move(AliasInfo);
402 }
403
404 // Remove the aliases collected this round from the RequestedAliases map.
405 for (auto &KV : QueryAliases)
406 RequestedAliases.erase(Val: KV.first);
407
408 assert(!QuerySymbols.empty() && "Alias cycle detected!");
409
410 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
411 if (!NewR) {
412 ES.reportError(Err: NewR.takeError());
413 R->failMaterialization();
414 return;
415 }
416
417 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
418 args: std::move(QueryAliases));
419 QueryInfos.push_back(
420 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
421 }
422
423 // Issue the queries.
424 while (!QueryInfos.empty()) {
425 auto QuerySymbols = std::move(QueryInfos.back().first);
426 auto QueryInfo = std::move(QueryInfos.back().second);
427
428 QueryInfos.pop_back();
429
430 auto RegisterDependencies = [QueryInfo,
431 &SrcJD](const SymbolDependenceMap &Deps) {
432 // If there were no materializing symbols, just bail out.
433 if (Deps.empty())
434 return;
435
436 // Otherwise the only deps should be on SrcJD.
437 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
438 "Unexpected dependencies for reexports");
439
440 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
441
442 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
443 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
444 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
445 };
446
447 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
448 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
449 if (Result) {
450 SymbolMap ResolutionMap;
451 for (auto &KV : QueryInfo->Aliases) {
452 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
453 Result->count(KV.second.Aliasee)) &&
454 "Result map missing entry?");
455 // Don't try to resolve materialization-side-effects-only symbols.
456 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
457 continue;
458
459 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
460 KV.second.AliasFlags};
461 }
462 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
463 ES.reportError(Err: std::move(Err));
464 QueryInfo->R->failMaterialization();
465 return;
466 }
467 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
468 ES.reportError(Err: std::move(Err));
469 QueryInfo->R->failMaterialization();
470 return;
471 }
472 } else {
473 ES.reportError(Err: Result.takeError());
474 QueryInfo->R->failMaterialization();
475 }
476 };
477
478 ES.lookup(K: LookupKind::Static,
479 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
480 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
481 RegisterDependencies: std::move(RegisterDependencies));
482 }
483}
484
485void ReExportsMaterializationUnit::discard(const JITDylib &JD,
486 const SymbolStringPtr &Name) {
487 assert(Aliases.count(Name) &&
488 "Symbol not covered by this MaterializationUnit");
489 Aliases.erase(Val: Name);
490}
491
492MaterializationUnit::Interface
493ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
494 SymbolFlagsMap SymbolFlags;
495 for (auto &KV : Aliases)
496 SymbolFlags[KV.first] = KV.second.AliasFlags;
497
498 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
499}
500
501Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
502 SymbolNameSet Symbols) {
503 SymbolLookupSet LookupSet(Symbols);
504 auto Flags = SourceJD.getExecutionSession().lookupFlags(
505 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
506 Symbols: SymbolLookupSet(std::move(Symbols)));
507
508 if (!Flags)
509 return Flags.takeError();
510
511 SymbolAliasMap Result;
512 for (auto &Name : Symbols) {
513 assert(Flags->count(Name) && "Missing entry in flags map");
514 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
515 }
516
517 return Result;
518}
519
520class InProgressLookupState {
521public:
522 // FIXME: Reduce the number of SymbolStringPtrs here. See
523 // https://github.com/llvm/llvm-project/issues/55576.
524
525 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
526 SymbolLookupSet LookupSet, SymbolState RequiredState)
527 : K(K), SearchOrder(std::move(SearchOrder)),
528 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
529 DefGeneratorCandidates = this->LookupSet;
530 }
531 virtual ~InProgressLookupState() = default;
532 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
533 virtual void fail(Error Err) = 0;
534
535 LookupKind K;
536 JITDylibSearchOrder SearchOrder;
537 SymbolLookupSet LookupSet;
538 SymbolState RequiredState;
539
540 size_t CurSearchOrderIndex = 0;
541 bool NewJITDylib = true;
542 SymbolLookupSet DefGeneratorCandidates;
543 SymbolLookupSet DefGeneratorNonCandidates;
544
545 enum {
546 NotInGenerator, // Not currently using a generator.
547 ResumedForGenerator, // Resumed after being auto-suspended before generator.
548 InGenerator // Currently using generator.
549 } GenState = NotInGenerator;
550 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
551};
552
553class InProgressLookupFlagsState : public InProgressLookupState {
554public:
555 InProgressLookupFlagsState(
556 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
557 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
558 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
559 SymbolState::NeverSearched),
560 OnComplete(std::move(OnComplete)) {}
561
562 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
563 auto &ES = SearchOrder.front().first->getExecutionSession();
564 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
565 }
566
567 void fail(Error Err) override { OnComplete(std::move(Err)); }
568
569private:
570 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
571};
572
573class InProgressFullLookupState : public InProgressLookupState {
574public:
575 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
576 SymbolLookupSet LookupSet,
577 SymbolState RequiredState,
578 std::shared_ptr<AsynchronousSymbolQuery> Q,
579 RegisterDependenciesFunction RegisterDependencies)
580 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
581 RequiredState),
582 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
583 }
584
585 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
586 auto &ES = SearchOrder.front().first->getExecutionSession();
587 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
588 RegisterDependencies: std::move(RegisterDependencies));
589 }
590
591 void fail(Error Err) override {
592 Q->detach();
593 Q->handleFailed(Err: std::move(Err));
594 }
595
596private:
597 std::shared_ptr<AsynchronousSymbolQuery> Q;
598 RegisterDependenciesFunction RegisterDependencies;
599};
600
601ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
602 JITDylibLookupFlags SourceJDLookupFlags,
603 SymbolPredicate Allow)
604 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
605 Allow(std::move(Allow)) {}
606
607Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
608 JITDylib &JD,
609 JITDylibLookupFlags JDLookupFlags,
610 const SymbolLookupSet &LookupSet) {
611 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
612
613 // Use lookupFlags to find the subset of symbols that match our lookup.
614 auto Flags = JD.getExecutionSession().lookupFlags(
615 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
616 if (!Flags)
617 return Flags.takeError();
618
619 // Create an alias map.
620 orc::SymbolAliasMap AliasMap;
621 for (auto &KV : *Flags)
622 if (!Allow || Allow(KV.first))
623 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
624
625 if (AliasMap.empty())
626 return Error::success();
627
628 // Define the re-exports.
629 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
630}
631
632LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
633 : IPLS(std::move(IPLS)) {}
634
635void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
636
637LookupState::LookupState() = default;
638LookupState::LookupState(LookupState &&) = default;
639LookupState &LookupState::operator=(LookupState &&) = default;
640LookupState::~LookupState() = default;
641
642void LookupState::continueLookup(Error Err) {
643 assert(IPLS && "Cannot call continueLookup on empty LookupState");
644 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
645 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
646}
647
648DefinitionGenerator::~DefinitionGenerator() {
649 std::deque<LookupState> LookupsToFail;
650 {
651 std::lock_guard<std::mutex> Lock(M);
652 std::swap(x&: PendingLookups, y&: LookupsToFail);
653 InUse = false;
654 }
655
656 for (auto &LS : LookupsToFail)
657 LS.continueLookup(Err: make_error<StringError>(
658 Args: "Query waiting on DefinitionGenerator that was destroyed",
659 Args: inconvertibleErrorCode()));
660}
661
662JITDylib::~JITDylib() {
663 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
664}
665
666Error JITDylib::clear() {
667 std::vector<ResourceTrackerSP> TrackersToRemove;
668 ES.runSessionLocked(F: [&]() {
669 assert(State != Closed && "JD is defunct");
670 for (auto &KV : TrackerSymbols)
671 TrackersToRemove.push_back(x: KV.first);
672 TrackersToRemove.push_back(x: getDefaultResourceTracker());
673 });
674
675 Error Err = Error::success();
676 for (auto &RT : TrackersToRemove)
677 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
678 return Err;
679}
680
681ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
682 return ES.runSessionLocked(F: [this] {
683 assert(State != Closed && "JD is defunct");
684 if (!DefaultTracker)
685 DefaultTracker = new ResourceTracker(this);
686 return DefaultTracker;
687 });
688}
689
690ResourceTrackerSP JITDylib::createResourceTracker() {
691 return ES.runSessionLocked(F: [this] {
692 assert(State == Open && "JD is defunct");
693 ResourceTrackerSP RT = new ResourceTracker(this);
694 return RT;
695 });
696}
697
698void JITDylib::removeGenerator(DefinitionGenerator &G) {
699 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
700 // session lock (since it may have to send errors to pending queries).
701 std::shared_ptr<DefinitionGenerator> TmpDG;
702
703 ES.runSessionLocked(F: [&] {
704 assert(State == Open && "JD is defunct");
705 auto I = llvm::find_if(Range&: DefGenerators,
706 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
707 return H.get() == &G;
708 });
709 assert(I != DefGenerators.end() && "Generator not found");
710 TmpDG = std::move(*I);
711 DefGenerators.erase(position: I);
712 });
713}
714
715Expected<SymbolFlagsMap>
716JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
717 SymbolFlagsMap SymbolFlags) {
718
719 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
720 if (FromMR.RT->isDefunct())
721 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
722
723 std::vector<NonOwningSymbolStringPtr> AddedSyms;
724 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
725
726 for (auto &[Name, Flags] : SymbolFlags) {
727 auto EntryItr = Symbols.find(Val: Name);
728
729 // If the entry already exists...
730 if (EntryItr != Symbols.end()) {
731
732 // If this is a strong definition then error out.
733 if (!Flags.isWeak()) {
734 // Remove any symbols already added.
735 for (auto &S : AddedSyms)
736 Symbols.erase(I: Symbols.find_as(Val: S));
737
738 // FIXME: Return all duplicates.
739 return make_error<DuplicateDefinition>(
740 Args: std::string(*Name), Args: "defineMaterializing operation");
741 }
742
743 // Otherwise just make a note to discard this symbol after the loop.
744 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
745 continue;
746 } else
747 EntryItr =
748 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
749
750 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
751 EntryItr->second.setState(SymbolState::Materializing);
752 }
753
754 // Remove any rejected weak definitions from the SymbolFlags map.
755 while (!RejectedWeakDefs.empty()) {
756 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
757 RejectedWeakDefs.pop_back();
758 }
759
760 return SymbolFlags;
761 });
762}
763
764Error JITDylib::replace(MaterializationResponsibility &FromMR,
765 std::unique_ptr<MaterializationUnit> MU) {
766 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
767 std::unique_ptr<MaterializationUnit> MustRunMU;
768 std::unique_ptr<MaterializationResponsibility> MustRunMR;
769
770 auto Err =
771 ES.runSessionLocked(F: [&, this]() -> Error {
772 if (FromMR.RT->isDefunct())
773 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
774
775#ifndef NDEBUG
776 for (auto &KV : MU->getSymbols()) {
777 auto SymI = Symbols.find(KV.first);
778 assert(SymI != Symbols.end() && "Replacing unknown symbol");
779 assert(SymI->second.getState() == SymbolState::Materializing &&
780 "Can not replace a symbol that ha is not materializing");
781 assert(!SymI->second.hasMaterializerAttached() &&
782 "Symbol should not have materializer attached already");
783 assert(UnmaterializedInfos.count(KV.first) == 0 &&
784 "Symbol being replaced should have no UnmaterializedInfo");
785 }
786#endif // NDEBUG
787
788 // If the tracker is defunct we need to bail out immediately.
789
790 // If any symbol has pending queries against it then we need to
791 // materialize MU immediately.
792 for (auto &KV : MU->getSymbols()) {
793 auto MII = MaterializingInfos.find(Val: KV.first);
794 if (MII != MaterializingInfos.end()) {
795 if (MII->second.hasQueriesPending()) {
796 MustRunMR = ES.createMaterializationResponsibility(
797 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
798 InitSymbol: std::move(MU->InitSymbol));
799 MustRunMU = std::move(MU);
800 return Error::success();
801 }
802 }
803 }
804
805 // Otherwise, make MU responsible for all the symbols.
806 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
807 args: FromMR.RT.get());
808 for (auto &KV : UMI->MU->getSymbols()) {
809 auto SymI = Symbols.find(Val: KV.first);
810 assert(SymI->second.getState() == SymbolState::Materializing &&
811 "Can not replace a symbol that is not materializing");
812 assert(!SymI->second.hasMaterializerAttached() &&
813 "Can not replace a symbol that has a materializer attached");
814 assert(UnmaterializedInfos.count(KV.first) == 0 &&
815 "Unexpected materializer entry in map");
816 SymI->second.setAddress(SymI->second.getAddress());
817 SymI->second.setMaterializerAttached(true);
818
819 auto &UMIEntry = UnmaterializedInfos[KV.first];
820 assert((!UMIEntry || !UMIEntry->MU) &&
821 "Replacing symbol with materializer still attached");
822 UMIEntry = UMI;
823 }
824
825 return Error::success();
826 });
827
828 if (Err)
829 return Err;
830
831 if (MustRunMU) {
832 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
833 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
834 args: std::move(MustRunMU), args: std::move(MustRunMR)));
835 } else {
836 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
837 }
838
839 return Error::success();
840}
841
842Expected<std::unique_ptr<MaterializationResponsibility>>
843JITDylib::delegate(MaterializationResponsibility &FromMR,
844 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
845
846 return ES.runSessionLocked(
847 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
848 if (FromMR.RT->isDefunct())
849 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
850
851 return ES.createMaterializationResponsibility(
852 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
853 });
854}
855
856SymbolNameSet
857JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
858 return ES.runSessionLocked(F: [&]() {
859 SymbolNameSet RequestedSymbols;
860
861 for (auto &KV : SymbolFlags) {
862 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
863 assert(Symbols.find(KV.first)->second.getState() !=
864 SymbolState::NeverSearched &&
865 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
866 "getRequestedSymbols can only be called for symbols that have "
867 "started materializing");
868 auto I = MaterializingInfos.find(Val: KV.first);
869 if (I == MaterializingInfos.end())
870 continue;
871
872 if (I->second.hasQueriesPending())
873 RequestedSymbols.insert(V: KV.first);
874 }
875
876 return RequestedSymbols;
877 });
878}
879
880Error JITDylib::resolve(MaterializationResponsibility &MR,
881 const SymbolMap &Resolved) {
882 AsynchronousSymbolQuerySet CompletedQueries;
883
884 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
885 if (MR.RT->isDefunct())
886 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
887
888 if (State != Open)
889 return make_error<StringError>(Args: "JITDylib " + getName() +
890 " is defunct",
891 Args: inconvertibleErrorCode());
892
893 struct WorklistEntry {
894 SymbolTable::iterator SymI;
895 ExecutorSymbolDef ResolvedSym;
896 };
897
898 SymbolNameSet SymbolsInErrorState;
899 std::vector<WorklistEntry> Worklist;
900 Worklist.reserve(n: Resolved.size());
901
902 // Build worklist and check for any symbols in the error state.
903 for (const auto &KV : Resolved) {
904
905 assert(!KV.second.getFlags().hasError() &&
906 "Resolution result can not have error flag set");
907
908 auto SymI = Symbols.find(Val: KV.first);
909
910 assert(SymI != Symbols.end() && "Symbol not found");
911 assert(!SymI->second.hasMaterializerAttached() &&
912 "Resolving symbol with materializer attached?");
913 assert(SymI->second.getState() == SymbolState::Materializing &&
914 "Symbol should be materializing");
915 assert(SymI->second.getAddress() == ExecutorAddr() &&
916 "Symbol has already been resolved");
917
918 if (SymI->second.getFlags().hasError())
919 SymbolsInErrorState.insert(V: KV.first);
920 else {
921 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
922 [[maybe_unused]] auto WeakOrCommon =
923 JITSymbolFlags::Weak | JITSymbolFlags::Common;
924 assert((KV.second.getFlags() & WeakOrCommon) &&
925 "Common symbols must be resolved as common or weak");
926 assert((KV.second.getFlags() & ~WeakOrCommon) ==
927 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
928 "Resolving symbol with incorrect flags");
929
930 } else
931 assert(KV.second.getFlags() == SymI->second.getFlags() &&
932 "Resolved flags should match the declared flags");
933
934 Worklist.push_back(
935 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
936 }
937 }
938
939 // If any symbols were in the error state then bail out.
940 if (!SymbolsInErrorState.empty()) {
941 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
942 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
943 return make_error<FailedToMaterialize>(
944 Args: getExecutionSession().getSymbolStringPool(),
945 Args: std::move(FailedSymbolsDepMap));
946 }
947
948 while (!Worklist.empty()) {
949 auto SymI = Worklist.back().SymI;
950 auto ResolvedSym = Worklist.back().ResolvedSym;
951 Worklist.pop_back();
952
953 auto &Name = SymI->first;
954
955 // Resolved symbols can not be weak: discard the weak flag.
956 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
957 SymI->second.setAddress(ResolvedSym.getAddress());
958 SymI->second.setFlags(ResolvedFlags);
959 SymI->second.setState(SymbolState::Resolved);
960
961 auto MII = MaterializingInfos.find(Val: Name);
962 if (MII == MaterializingInfos.end())
963 continue;
964
965 auto &MI = MII->second;
966 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
967 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
968 if (Q->isComplete())
969 CompletedQueries.insert(x: std::move(Q));
970 }
971 }
972
973 return Error::success();
974 }))
975 return Err;
976
977 // Otherwise notify all the completed queries.
978 for (auto &Q : CompletedQueries) {
979 assert(Q->isComplete() && "Q not completed");
980 Q->handleComplete(ES);
981 }
982
983 return Error::success();
984}
985
986void JITDylib::unlinkMaterializationResponsibility(
987 MaterializationResponsibility &MR) {
988 ES.runSessionLocked(F: [&]() {
989 auto I = TrackerMRs.find(Val: MR.RT.get());
990 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
991 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
992 I->second.erase(V: &MR);
993 if (I->second.empty())
994 TrackerMRs.erase(Val: MR.RT.get());
995 });
996}
997
998void JITDylib::shrinkMaterializationInfoMemory() {
999 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1000 // memory since we may have long-lived JDs after linking is done.
1001
1002 if (UnmaterializedInfos.empty())
1003 UnmaterializedInfos.clear();
1004
1005 if (MaterializingInfos.empty())
1006 MaterializingInfos.clear();
1007}
1008
1009void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1010 bool LinkAgainstThisJITDylibFirst) {
1011 ES.runSessionLocked(F: [&]() {
1012 assert(State == Open && "JD is defunct");
1013 if (LinkAgainstThisJITDylibFirst) {
1014 LinkOrder.clear();
1015 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1016 LinkOrder.push_back(
1017 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1018 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1019 } else
1020 LinkOrder = std::move(NewLinkOrder);
1021 });
1022}
1023
1024void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1025 ES.runSessionLocked(F: [&]() {
1026 for (auto &KV : NewLinks) {
1027 // Skip elements of NewLinks that are already in the link order.
1028 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1029 continue;
1030
1031 LinkOrder.push_back(x: std::move(KV));
1032 }
1033 });
1034}
1035
1036void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1037 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1038}
1039
1040void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1041 JITDylibLookupFlags JDLookupFlags) {
1042 ES.runSessionLocked(F: [&]() {
1043 assert(State == Open && "JD is defunct");
1044 for (auto &KV : LinkOrder)
1045 if (KV.first == &OldJD) {
1046 KV = {&NewJD, JDLookupFlags};
1047 break;
1048 }
1049 });
1050}
1051
1052void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1053 ES.runSessionLocked(F: [&]() {
1054 assert(State == Open && "JD is defunct");
1055 auto I = llvm::find_if(Range&: LinkOrder,
1056 P: [&](const JITDylibSearchOrder::value_type &KV) {
1057 return KV.first == &JD;
1058 });
1059 if (I != LinkOrder.end())
1060 LinkOrder.erase(position: I);
1061 });
1062}
1063
1064Error JITDylib::remove(const SymbolNameSet &Names) {
1065 return ES.runSessionLocked(F: [&]() -> Error {
1066 assert(State == Open && "JD is defunct");
1067 using SymbolMaterializerItrPair =
1068 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1069 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1070 SymbolNameSet Missing;
1071 SymbolNameSet Materializing;
1072
1073 for (auto &Name : Names) {
1074 auto I = Symbols.find(Val: Name);
1075
1076 // Note symbol missing.
1077 if (I == Symbols.end()) {
1078 Missing.insert(V: Name);
1079 continue;
1080 }
1081
1082 // Note symbol materializing.
1083 if (I->second.getState() != SymbolState::NeverSearched &&
1084 I->second.getState() != SymbolState::Ready) {
1085 Materializing.insert(V: Name);
1086 continue;
1087 }
1088
1089 auto UMII = I->second.hasMaterializerAttached()
1090 ? UnmaterializedInfos.find(Val: Name)
1091 : UnmaterializedInfos.end();
1092 SymbolsToRemove.push_back(x: std::make_pair(x&: I, y&: UMII));
1093 }
1094
1095 // If any of the symbols are not defined, return an error.
1096 if (!Missing.empty())
1097 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1098 Args: std::move(Missing));
1099
1100 // If any of the symbols are currently materializing, return an error.
1101 if (!Materializing.empty())
1102 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1103 Args: std::move(Materializing));
1104
1105 // Remove the symbols.
1106 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1107 auto UMII = SymbolMaterializerItrPair.second;
1108
1109 // If there is a materializer attached, call discard.
1110 if (UMII != UnmaterializedInfos.end()) {
1111 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1112 UnmaterializedInfos.erase(I: UMII);
1113 }
1114
1115 auto SymI = SymbolMaterializerItrPair.first;
1116 Symbols.erase(I: SymI);
1117 }
1118
1119 shrinkMaterializationInfoMemory();
1120
1121 return Error::success();
1122 });
1123}
1124
1125void JITDylib::dump(raw_ostream &OS) {
1126 ES.runSessionLocked(F: [&, this]() {
1127 OS << "JITDylib \"" << getName() << "\" (ES: "
1128 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1129 << ", State = ";
1130 switch (State) {
1131 case Open:
1132 OS << "Open";
1133 break;
1134 case Closing:
1135 OS << "Closing";
1136 break;
1137 case Closed:
1138 OS << "Closed";
1139 break;
1140 }
1141 OS << ")\n";
1142 if (State == Closed)
1143 return;
1144 OS << "Link order: " << LinkOrder << "\n"
1145 << "Symbol table:\n";
1146
1147 // Sort symbols so we get a deterministic order and can check them in tests.
1148 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1149 for (auto &KV : Symbols)
1150 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1151 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1152 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1153
1154 for (auto &KV : SymbolsSorted) {
1155 OS << " \"" << *KV.first << "\": ";
1156 if (auto Addr = KV.second->getAddress())
1157 OS << Addr;
1158 else
1159 OS << "<not resolved> ";
1160
1161 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1162
1163 if (KV.second->hasMaterializerAttached()) {
1164 OS << " (Materializer ";
1165 auto I = UnmaterializedInfos.find(Val: KV.first);
1166 assert(I != UnmaterializedInfos.end() &&
1167 "Lazy symbol should have UnmaterializedInfo");
1168 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1169 } else
1170 OS << "\n";
1171 }
1172
1173 if (!MaterializingInfos.empty())
1174 OS << " MaterializingInfos entries:\n";
1175 for (auto &KV : MaterializingInfos) {
1176 OS << " \"" << *KV.first << "\":\n"
1177 << " " << KV.second.pendingQueries().size()
1178 << " pending queries: { ";
1179 for (const auto &Q : KV.second.pendingQueries())
1180 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1181 OS << "}\n";
1182 }
1183 });
1184}
1185
1186void JITDylib::MaterializingInfo::addQuery(
1187 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1188
1189 auto I = llvm::lower_bound(
1190 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1191 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1192 return V->getRequiredState() <= S;
1193 });
1194 PendingQueries.insert(position: I.base(), x: std::move(Q));
1195}
1196
1197void JITDylib::MaterializingInfo::removeQuery(
1198 const AsynchronousSymbolQuery &Q) {
1199 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1200 auto I = llvm::find_if(
1201 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1202 return V.get() == &Q;
1203 });
1204 if (I != PendingQueries.end())
1205 PendingQueries.erase(position: I);
1206}
1207
1208JITDylib::AsynchronousSymbolQueryList
1209JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1210 AsynchronousSymbolQueryList Result;
1211 while (!PendingQueries.empty()) {
1212 if (PendingQueries.back()->getRequiredState() > RequiredState)
1213 break;
1214
1215 Result.push_back(x: std::move(PendingQueries.back()));
1216 PendingQueries.pop_back();
1217 }
1218
1219 return Result;
1220}
1221
1222JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1223 : JITLinkDylib(std::move(Name)), ES(ES) {
1224 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1225}
1226
1227JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1228 // Note: Should be called under the session lock.
1229 assert(State != Closed && "JD is defunct");
1230
1231 SymbolNameVector SymbolsToRemove;
1232 SymbolNameVector SymbolsToFail;
1233
1234 if (&RT == DefaultTracker.get()) {
1235 SymbolNameSet TrackedSymbols;
1236 for (auto &KV : TrackerSymbols)
1237 TrackedSymbols.insert_range(R&: KV.second);
1238
1239 for (auto &KV : Symbols) {
1240 auto &Sym = KV.first;
1241 if (!TrackedSymbols.count(V: Sym))
1242 SymbolsToRemove.push_back(x: Sym);
1243 }
1244
1245 DefaultTracker.reset();
1246 } else {
1247 /// Check for a non-default tracker.
1248 auto I = TrackerSymbols.find(Val: &RT);
1249 if (I != TrackerSymbols.end()) {
1250 SymbolsToRemove = std::move(I->second);
1251 TrackerSymbols.erase(I);
1252 }
1253 // ... if not found this tracker was already defunct. Nothing to do.
1254 }
1255
1256 for (auto &Sym : SymbolsToRemove) {
1257 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1258
1259 // If there is a MaterializingInfo then collect any queries to fail.
1260 auto MII = MaterializingInfos.find(Val: Sym);
1261 if (MII != MaterializingInfos.end())
1262 SymbolsToFail.push_back(x: Sym);
1263 }
1264
1265 auto [QueriesToFail, FailedSymbols] =
1266 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1267
1268 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1269
1270 // Removed symbols should be taken out of the table altogether.
1271 for (auto &Sym : SymbolsToRemove) {
1272 auto I = Symbols.find(Val: Sym);
1273 assert(I != Symbols.end() && "Symbol not present in table");
1274
1275 // Remove Materializer if present.
1276 if (I->second.hasMaterializerAttached()) {
1277 // FIXME: Should this discard the symbols?
1278 auto J = UnmaterializedInfos.find(Val: Sym);
1279 assert(J != UnmaterializedInfos.end() &&
1280 "Symbol table indicates MU present, but no UMI record");
1281 if (J->second->MU)
1282 DefunctMUs.push_back(x: std::move(J->second->MU));
1283 UnmaterializedInfos.erase(I: J);
1284 } else {
1285 assert(!UnmaterializedInfos.count(Sym) &&
1286 "Symbol has materializer attached");
1287 }
1288
1289 Symbols.erase(I);
1290 }
1291
1292 shrinkMaterializationInfoMemory();
1293
1294 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1295 .DefunctMUs: std::move(DefunctMUs)};
1296}
1297
1298void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1299 assert(State != Closed && "JD is defunct");
1300 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1301 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1302 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1303
1304 // Update trackers for any not-yet materialized units.
1305 for (auto &KV : UnmaterializedInfos) {
1306 if (KV.second->RT == &SrcRT)
1307 KV.second->RT = &DstRT;
1308 }
1309
1310 // Update trackers for any active materialization responsibilities.
1311 {
1312 auto I = TrackerMRs.find(Val: &SrcRT);
1313 if (I != TrackerMRs.end()) {
1314 auto &SrcMRs = I->second;
1315 auto &DstMRs = TrackerMRs[&DstRT];
1316 for (auto *MR : SrcMRs)
1317 MR->RT = &DstRT;
1318 if (DstMRs.empty())
1319 DstMRs = std::move(SrcMRs);
1320 else
1321 DstMRs.insert_range(R&: SrcMRs);
1322 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1323 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1324 TrackerMRs.erase(Val: &SrcRT);
1325 }
1326 }
1327
1328 // If we're transfering to the default tracker we just need to delete the
1329 // tracked symbols for the source tracker.
1330 if (&DstRT == DefaultTracker.get()) {
1331 TrackerSymbols.erase(Val: &SrcRT);
1332 return;
1333 }
1334
1335 // If we're transferring from the default tracker we need to find all
1336 // currently untracked symbols.
1337 if (&SrcRT == DefaultTracker.get()) {
1338 assert(!TrackerSymbols.count(&SrcRT) &&
1339 "Default tracker should not appear in TrackerSymbols");
1340
1341 SymbolNameVector SymbolsToTrack;
1342
1343 SymbolNameSet CurrentlyTrackedSymbols;
1344 for (auto &KV : TrackerSymbols)
1345 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1346
1347 for (auto &KV : Symbols) {
1348 auto &Sym = KV.first;
1349 if (!CurrentlyTrackedSymbols.count(V: Sym))
1350 SymbolsToTrack.push_back(x: Sym);
1351 }
1352
1353 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1354 return;
1355 }
1356
1357 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1358
1359 // Finally if neither SrtRT or DstRT are the default tracker then
1360 // just append DstRT's tracked symbols to SrtRT's.
1361 auto SI = TrackerSymbols.find(Val: &SrcRT);
1362 if (SI == TrackerSymbols.end())
1363 return;
1364
1365 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1366 for (auto &Sym : SI->second)
1367 DstTrackedSymbols.push_back(x: std::move(Sym));
1368 TrackerSymbols.erase(I: SI);
1369}
1370
1371Error JITDylib::defineImpl(MaterializationUnit &MU) {
1372 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1373
1374 SymbolNameSet Duplicates;
1375 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1376 std::vector<SymbolStringPtr> MUDefsOverridden;
1377
1378 for (const auto &KV : MU.getSymbols()) {
1379 auto I = Symbols.find(Val: KV.first);
1380
1381 if (I != Symbols.end()) {
1382 if (KV.second.isStrong()) {
1383 if (I->second.getFlags().isStrong() ||
1384 I->second.getState() > SymbolState::NeverSearched)
1385 Duplicates.insert(V: KV.first);
1386 else {
1387 assert(I->second.getState() == SymbolState::NeverSearched &&
1388 "Overridden existing def should be in the never-searched "
1389 "state");
1390 ExistingDefsOverridden.push_back(x: KV.first);
1391 }
1392 } else
1393 MUDefsOverridden.push_back(x: KV.first);
1394 }
1395 }
1396
1397 // If there were any duplicate definitions then bail out.
1398 if (!Duplicates.empty()) {
1399 LLVM_DEBUG(
1400 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1401 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1402 Args: MU.getName().str());
1403 }
1404
1405 // Discard any overridden defs in this MU.
1406 LLVM_DEBUG({
1407 if (!MUDefsOverridden.empty())
1408 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1409 });
1410 for (auto &S : MUDefsOverridden)
1411 MU.doDiscard(JD: *this, Name: S);
1412
1413 // Discard existing overridden defs.
1414 LLVM_DEBUG({
1415 if (!ExistingDefsOverridden.empty())
1416 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1417 << "\n";
1418 });
1419 for (auto &S : ExistingDefsOverridden) {
1420
1421 auto UMII = UnmaterializedInfos.find(Val: S);
1422 assert(UMII != UnmaterializedInfos.end() &&
1423 "Overridden existing def should have an UnmaterializedInfo");
1424 UMII->second->MU->doDiscard(JD: *this, Name: S);
1425 }
1426
1427 // Finally, add the defs from this MU.
1428 for (auto &KV : MU.getSymbols()) {
1429 auto &SymEntry = Symbols[KV.first];
1430 SymEntry.setFlags(KV.second);
1431 SymEntry.setState(SymbolState::NeverSearched);
1432 SymEntry.setMaterializerAttached(true);
1433 }
1434
1435 return Error::success();
1436}
1437
1438void JITDylib::installMaterializationUnit(
1439 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1440
1441 /// defineImpl succeeded.
1442 if (&RT != DefaultTracker.get()) {
1443 auto &TS = TrackerSymbols[&RT];
1444 TS.reserve(n: TS.size() + MU->getSymbols().size());
1445 for (auto &KV : MU->getSymbols())
1446 TS.push_back(x: KV.first);
1447 }
1448
1449 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1450 for (auto &KV : UMI->MU->getSymbols())
1451 UnmaterializedInfos[KV.first] = UMI;
1452}
1453
1454void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1455 const SymbolNameSet &QuerySymbols) {
1456 for (auto &QuerySymbol : QuerySymbols) {
1457 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1458 if (MII != MaterializingInfos.end())
1459 MII->second.removeQuery(Q);
1460 }
1461}
1462
1463Platform::~Platform() = default;
1464
1465Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1466 ExecutionSession &ES,
1467 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1468
1469 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1470 Error CompoundErr = Error::success();
1471 std::mutex LookupMutex;
1472 std::condition_variable CV;
1473 uint64_t Count = InitSyms.size();
1474
1475 LLVM_DEBUG({
1476 dbgs() << "Issuing init-symbol lookup:\n";
1477 for (auto &KV : InitSyms)
1478 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1479 });
1480
1481 for (auto &KV : InitSyms) {
1482 auto *JD = KV.first;
1483 auto Names = std::move(KV.second);
1484 ES.lookup(
1485 K: LookupKind::Static,
1486 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1487 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1488 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1489 {
1490 std::lock_guard<std::mutex> Lock(LookupMutex);
1491 --Count;
1492 if (Result) {
1493 assert(!CompoundResult.count(JD) &&
1494 "Duplicate JITDylib in lookup?");
1495 CompoundResult[JD] = std::move(*Result);
1496 } else
1497 CompoundErr =
1498 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1499 }
1500 CV.notify_one();
1501 },
1502 RegisterDependencies: NoDependenciesToRegister);
1503 }
1504
1505 std::unique_lock<std::mutex> Lock(LookupMutex);
1506 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1507
1508 if (CompoundErr)
1509 return std::move(CompoundErr);
1510
1511 return std::move(CompoundResult);
1512}
1513
1514void Platform::lookupInitSymbolsAsync(
1515 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1516 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1517
1518 class TriggerOnComplete {
1519 public:
1520 using OnCompleteFn = unique_function<void(Error)>;
1521 TriggerOnComplete(OnCompleteFn OnComplete)
1522 : OnComplete(std::move(OnComplete)) {}
1523 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1524 void reportResult(Error Err) {
1525 std::lock_guard<std::mutex> Lock(ResultMutex);
1526 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1527 }
1528
1529 private:
1530 std::mutex ResultMutex;
1531 Error LookupResult{Error::success()};
1532 OnCompleteFn OnComplete;
1533 };
1534
1535 LLVM_DEBUG({
1536 dbgs() << "Issuing init-symbol lookup:\n";
1537 for (auto &KV : InitSyms)
1538 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1539 });
1540
1541 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1542
1543 for (auto &KV : InitSyms) {
1544 auto *JD = KV.first;
1545 auto Names = std::move(KV.second);
1546 ES.lookup(
1547 K: LookupKind::Static,
1548 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1549 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1550 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1551 TOC->reportResult(Err: Result.takeError());
1552 },
1553 RegisterDependencies: NoDependenciesToRegister);
1554 }
1555}
1556
1557MaterializationTask::~MaterializationTask() {
1558 // If this task wasn't run then fail materialization.
1559 if (MR)
1560 MR->failMaterialization();
1561}
1562
1563void MaterializationTask::printDescription(raw_ostream &OS) {
1564 OS << "Materialization task: " << MU->getName() << " in "
1565 << MR->getTargetJITDylib().getName();
1566}
1567
1568void MaterializationTask::run() {
1569 assert(MU && "MU should not be null");
1570 assert(MR && "MR should not be null");
1571 MU->materialize(R: std::move(MR));
1572}
1573
1574void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1575
1576void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1577
1578ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1579 : EPC(std::move(EPC)) {
1580 // Associated EPC and this.
1581 this->EPC->ES = this;
1582}
1583
1584ExecutionSession::~ExecutionSession() {
1585 // You must call endSession prior to destroying the session.
1586 assert(!SessionOpen &&
1587 "Session still open. Did you forget to call endSession?");
1588}
1589
1590Error ExecutionSession::endSession() {
1591 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1592
1593 auto JDsToRemove = runSessionLocked(F: [&] {
1594
1595#ifdef EXPENSIVE_CHECKS
1596 verifySessionState("Entering ExecutionSession::endSession");
1597#endif
1598
1599 SessionOpen = false;
1600 return JDs;
1601 });
1602
1603 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1604
1605 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1606
1607 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1608
1609 return Err;
1610}
1611
1612void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1613 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1614}
1615
1616void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1617 runSessionLocked(F: [&] {
1618 assert(!ResourceManagers.empty() && "No managers registered");
1619 if (ResourceManagers.back() == &RM)
1620 ResourceManagers.pop_back();
1621 else {
1622 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1623 assert(I != ResourceManagers.end() && "RM not registered");
1624 ResourceManagers.erase(position: I);
1625 }
1626 });
1627}
1628
1629JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1630 return runSessionLocked(F: [&, this]() -> JITDylib * {
1631 for (auto &JD : JDs)
1632 if (JD->getName() == Name)
1633 return JD.get();
1634 return nullptr;
1635 });
1636}
1637
1638JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1639 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1640 return runSessionLocked(F: [&, this]() -> JITDylib & {
1641 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1642 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1643 return *JDs.back();
1644 });
1645}
1646
1647Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1648 auto &JD = createBareJITDylib(Name);
1649 if (P)
1650 if (auto Err = P->setupJITDylib(JD))
1651 return std::move(Err);
1652 return JD;
1653}
1654
1655Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1656 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1657 runSessionLocked(F: [&] {
1658 for (auto &JD : JDsToRemove) {
1659 assert(JD->State == JITDylib::Open && "JD already closed");
1660 JD->State = JITDylib::Closing;
1661 auto I = llvm::find(Range&: JDs, Val: JD);
1662 assert(I != JDs.end() && "JD does not appear in session JDs");
1663 JDs.erase(position: I);
1664 }
1665 });
1666
1667 // Clear JITDylibs and notify the platform.
1668 Error Err = Error::success();
1669 for (auto JD : JDsToRemove) {
1670 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1671 if (P)
1672 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1673 }
1674
1675 // Set JD to closed state. Clear remaining data structures.
1676 runSessionLocked(F: [&] {
1677 for (auto &JD : JDsToRemove) {
1678 assert(JD->State == JITDylib::Closing && "JD should be closing");
1679 JD->State = JITDylib::Closed;
1680 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1681 assert(JD->UnmaterializedInfos.empty() &&
1682 "JD.UnmaterializedInfos is not empty after clear");
1683 assert(JD->MaterializingInfos.empty() &&
1684 "JD.MaterializingInfos is not empty after clear");
1685 assert(JD->TrackerSymbols.empty() &&
1686 "TrackerSymbols is not empty after clear");
1687 JD->DefGenerators.clear();
1688 JD->LinkOrder.clear();
1689 }
1690 });
1691
1692 return Err;
1693}
1694
1695Expected<std::vector<JITDylibSP>>
1696JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1697 if (JDs.empty())
1698 return std::vector<JITDylibSP>();
1699
1700 auto &ES = JDs.front()->getExecutionSession();
1701 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1702 DenseSet<JITDylib *> Visited;
1703 std::vector<JITDylibSP> Result;
1704
1705 for (auto &JD : JDs) {
1706
1707 if (JD->State != Open)
1708 return make_error<StringError>(
1709 Args: "Error building link order: " + JD->getName() + " is defunct",
1710 Args: inconvertibleErrorCode());
1711 if (Visited.count(V: JD.get()))
1712 continue;
1713
1714 SmallVector<JITDylibSP, 64> WorkStack;
1715 WorkStack.push_back(Elt: JD);
1716 Visited.insert(V: JD.get());
1717
1718 while (!WorkStack.empty()) {
1719 Result.push_back(x: std::move(WorkStack.back()));
1720 WorkStack.pop_back();
1721
1722 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1723 auto &JD = *KV.first;
1724 if (!Visited.insert(V: &JD).second)
1725 continue;
1726 WorkStack.push_back(Elt: &JD);
1727 }
1728 }
1729 }
1730 return Result;
1731 });
1732}
1733
1734Expected<std::vector<JITDylibSP>>
1735JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1736 auto Result = getDFSLinkOrder(JDs);
1737 if (Result)
1738 std::reverse(first: Result->begin(), last: Result->end());
1739 return Result;
1740}
1741
1742Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1743 return getDFSLinkOrder(JDs: {this});
1744}
1745
1746Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1747 return getReverseDFSLinkOrder(JDs: {this});
1748}
1749
1750void ExecutionSession::lookupFlags(
1751 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1752 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1753
1754 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1755 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1756 args: std::move(OnComplete)),
1757 Err: Error::success());
1758}
1759
1760Expected<SymbolFlagsMap>
1761ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1762 SymbolLookupSet LookupSet) {
1763
1764 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1765 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1766 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1767 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1768 ResultP.set_value(std::move(Result));
1769 }),
1770 Err: Error::success());
1771
1772 auto ResultF = ResultP.get_future();
1773 return ResultF.get();
1774}
1775
1776void ExecutionSession::lookup(
1777 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1778 SymbolLookupSet Symbols, SymbolState RequiredState,
1779 SymbolsResolvedCallback NotifyComplete,
1780 RegisterDependenciesFunction RegisterDependencies) {
1781
1782 LLVM_DEBUG({
1783 runSessionLocked([&]() {
1784 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1785 << " (required state: " << RequiredState << ")\n";
1786 });
1787 });
1788
1789 // lookup can be re-entered recursively if running on a single thread. Run any
1790 // outstanding MUs in case this query depends on them, otherwise this lookup
1791 // will starve waiting for a result from an MU that is stuck in the queue.
1792 dispatchOutstandingMUs();
1793
1794 auto Unresolved = std::move(Symbols);
1795 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1796 args: std::move(NotifyComplete));
1797
1798 auto IPLS = std::make_unique<InProgressFullLookupState>(
1799 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1800 args: std::move(RegisterDependencies));
1801
1802 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1803}
1804
1805Expected<SymbolMap>
1806ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1807 SymbolLookupSet Symbols, LookupKind K,
1808 SymbolState RequiredState,
1809 RegisterDependenciesFunction RegisterDependencies) {
1810#if LLVM_ENABLE_THREADS
1811 // In the threaded case we use promises to return the results.
1812 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1813
1814 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1815 PromisedResult.set_value(std::move(R));
1816 };
1817
1818#else
1819 SymbolMap Result;
1820 Error ResolutionError = Error::success();
1821
1822 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1823 ErrorAsOutParameter _(ResolutionError);
1824 if (R)
1825 Result = std::move(*R);
1826 else
1827 ResolutionError = R.takeError();
1828 };
1829#endif
1830
1831 // Perform the asynchronous lookup.
1832 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1833 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1834
1835#if LLVM_ENABLE_THREADS
1836 return PromisedResult.get_future().get();
1837#else
1838 if (ResolutionError)
1839 return std::move(ResolutionError);
1840
1841 return Result;
1842#endif
1843}
1844
1845Expected<ExecutorSymbolDef>
1846ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1847 SymbolStringPtr Name, SymbolState RequiredState) {
1848 SymbolLookupSet Names({Name});
1849
1850 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1851 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1852 assert(ResultMap->size() == 1 && "Unexpected number of results");
1853 assert(ResultMap->count(Name) && "Missing result for symbol");
1854 return std::move(ResultMap->begin()->second);
1855 } else
1856 return ResultMap.takeError();
1857}
1858
1859Expected<ExecutorSymbolDef>
1860ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1861 SymbolState RequiredState) {
1862 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1863}
1864
1865Expected<ExecutorSymbolDef>
1866ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1867 SymbolState RequiredState) {
1868 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1869}
1870
1871Error ExecutionSession::registerJITDispatchHandlers(
1872 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1873
1874 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1875 Symbols: SymbolLookupSet::fromMapKeys(
1876 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1877 if (!TagSyms)
1878 return TagSyms.takeError();
1879
1880 // Associate tag addresses with implementations.
1881 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1882
1883 // Check that no tags are being overwritten.
1884 for (auto &[TagName, TagSym] : *TagSyms) {
1885 auto TagAddr = TagSym.getAddress();
1886 if (JITDispatchHandlers.count(Val: TagAddr))
1887 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1888 " (for " + *TagName +
1889 ") already registered",
1890 Args: inconvertibleErrorCode());
1891 }
1892
1893 // At this point we're guaranteed to succeed. Install the handlers.
1894 for (auto &[TagName, TagSym] : *TagSyms) {
1895 auto TagAddr = TagSym.getAddress();
1896 auto I = WFs.find(Val: TagName);
1897 assert(I != WFs.end() && I->second &&
1898 "JITDispatchHandler implementation missing");
1899 JITDispatchHandlers[TagAddr] =
1900 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1901 LLVM_DEBUG({
1902 dbgs() << "Associated function tag \"" << *TagName << "\" ("
1903 << formatv("{0:x}", TagAddr) << ") with handler\n";
1904 });
1905 }
1906
1907 return Error::success();
1908}
1909
1910void ExecutionSession::runJITDispatchHandler(
1911 SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr,
1912 shared::WrapperFunctionBuffer ArgBytes) {
1913
1914 std::shared_ptr<JITDispatchHandlerFunction> F;
1915 {
1916 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1917 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1918 if (I != JITDispatchHandlers.end())
1919 F = I->second;
1920 }
1921
1922 if (F)
1923 (*F)(std::move(SendResult), ArgBytes.data(), ArgBytes.size());
1924 else
1925 SendResult(shared::WrapperFunctionBuffer::createOutOfBandError(
1926 Msg: ("No function registered for tag " +
1927 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1928 .str()));
1929}
1930
1931void ExecutionSession::dump(raw_ostream &OS) {
1932 runSessionLocked(F: [this, &OS]() {
1933 for (auto &JD : JDs)
1934 JD->dump(OS);
1935 });
1936}
1937
1938#ifdef EXPENSIVE_CHECKS
1939bool ExecutionSession::verifySessionState(Twine Phase) {
1940 return runSessionLocked([&]() {
1941 bool AllOk = true;
1942
1943 for (auto &JD : JDs) {
1944
1945 auto LogFailure = [&]() -> raw_fd_ostream & {
1946 auto &Stream = errs();
1947 if (AllOk)
1948 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1949 << "\n";
1950 Stream << " In JITDylib " << JD->getName() << ", ";
1951 AllOk = false;
1952 return Stream;
1953 };
1954
1955 if (JD->State != JITDylib::Open) {
1956 LogFailure()
1957 << "state is not Open, but JD is in ExecutionSession list.";
1958 }
1959
1960 // Check symbol table.
1961 // 1. If the entry state isn't resolved then check that no address has
1962 // been set.
1963 // 2. Check that if the hasMaterializerAttached flag is set then there is
1964 // an UnmaterializedInfo entry, and vice-versa.
1965 for (auto &[Sym, Entry] : JD->Symbols) {
1966 // Check that unresolved symbols have null addresses.
1967 if (Entry.getState() < SymbolState::Resolved) {
1968 if (Entry.getAddress()) {
1969 LogFailure() << "symbol " << Sym << " has state "
1970 << Entry.getState()
1971 << " (not-yet-resolved) but non-null address "
1972 << Entry.getAddress() << ".\n";
1973 }
1974 }
1975
1976 // Check that the hasMaterializerAttached flag is correct.
1977 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
1978 if (Entry.hasMaterializerAttached()) {
1979 if (UMIItr == JD->UnmaterializedInfos.end()) {
1980 LogFailure() << "symbol " << Sym
1981 << " entry claims materializer attached, but "
1982 "UnmaterializedInfos has no corresponding entry.\n";
1983 }
1984 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
1985 LogFailure()
1986 << "symbol " << Sym
1987 << " entry claims no materializer attached, but "
1988 "UnmaterializedInfos has an unexpected entry for it.\n";
1989 }
1990 }
1991
1992 // Check that every UnmaterializedInfo entry has a corresponding entry
1993 // in the Symbols table.
1994 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
1995 auto SymItr = JD->Symbols.find(Sym);
1996 if (SymItr == JD->Symbols.end()) {
1997 LogFailure()
1998 << "symbol " << Sym
1999 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2000 }
2001 }
2002
2003 // Check consistency of the MaterializingInfos table.
2004 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2005
2006 auto SymItr = JD->Symbols.find(Sym);
2007 if (SymItr == JD->Symbols.end()) {
2008 // If there's no Symbols entry for this MaterializingInfos entry then
2009 // report that.
2010 LogFailure()
2011 << "symbol " << Sym
2012 << " has MaterializingInfos entry, but no Symbols entry.\n";
2013 } else {
2014 // Otherwise check consistency between Symbols and MaterializingInfos.
2015
2016 // Ready symbols should not have MaterializingInfos.
2017 if (SymItr->second.getState() == SymbolState::Ready) {
2018 LogFailure()
2019 << "symbol " << Sym
2020 << " is in Ready state, should not have MaterializingInfo.\n";
2021 }
2022
2023 // Pending queries should be for subsequent states.
2024 auto CurState = static_cast<SymbolState>(
2025 static_cast<std::underlying_type_t<SymbolState>>(
2026 SymItr->second.getState()) + 1);
2027 for (auto &Q : MII.PendingQueries) {
2028 if (Q->getRequiredState() != CurState) {
2029 if (Q->getRequiredState() > CurState)
2030 CurState = Q->getRequiredState();
2031 else
2032 LogFailure() << "symbol " << Sym
2033 << " has stale or misordered queries.\n";
2034 }
2035 }
2036 }
2037 }
2038 }
2039
2040 return AllOk;
2041 });
2042}
2043#endif // EXPENSIVE_CHECKS
2044
2045void ExecutionSession::dispatchOutstandingMUs() {
2046 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2047 while (true) {
2048 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2049 std::unique_ptr<MaterializationResponsibility>>>
2050 JMU;
2051
2052 {
2053 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2054 if (!OutstandingMUs.empty()) {
2055 JMU.emplace(args: std::move(OutstandingMUs.back()));
2056 OutstandingMUs.pop_back();
2057 }
2058 }
2059
2060 if (!JMU)
2061 break;
2062
2063 assert(JMU->first && "No MU?");
2064 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2065 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2066 args: std::move(JMU->second)));
2067 }
2068 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2069}
2070
2071Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2072 LLVM_DEBUG({
2073 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2074 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2075 });
2076 std::vector<ResourceManager *> CurrentResourceManagers;
2077
2078 JITDylib::RemoveTrackerResult R;
2079
2080 runSessionLocked(F: [&] {
2081 CurrentResourceManagers = ResourceManagers;
2082 RT.makeDefunct();
2083 R = RT.getJITDylib().IL_removeTracker(RT);
2084 });
2085
2086 // Release any defunct MaterializationUnits.
2087 R.DefunctMUs.clear();
2088
2089 Error Err = Error::success();
2090
2091 auto &JD = RT.getJITDylib();
2092 for (auto *L : reverse(C&: CurrentResourceManagers))
2093 Err = joinErrors(E1: std::move(Err),
2094 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2095
2096 for (auto &Q : R.QueriesToFail)
2097 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2098 Args&: R.FailedSymbols));
2099
2100 return Err;
2101}
2102
2103void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2104 ResourceTracker &SrcRT) {
2105 LLVM_DEBUG({
2106 dbgs() << "In " << SrcRT.getJITDylib().getName()
2107 << " transfering resources from tracker "
2108 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2109 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2110 });
2111
2112 // No-op transfers are allowed and do not invalidate the source.
2113 if (&DstRT == &SrcRT)
2114 return;
2115
2116 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2117 "Can't transfer resources between JITDylibs");
2118 runSessionLocked(F: [&]() {
2119 SrcRT.makeDefunct();
2120 auto &JD = DstRT.getJITDylib();
2121 JD.transferTracker(DstRT, SrcRT);
2122 for (auto *L : reverse(C&: ResourceManagers))
2123 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2124 SrcK: SrcRT.getKeyUnsafe());
2125 });
2126}
2127
2128void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2129 runSessionLocked(F: [&]() {
2130 LLVM_DEBUG({
2131 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2132 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2133 });
2134 if (!RT.isDefunct())
2135 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2136 SrcRT&: RT);
2137 });
2138}
2139
2140Error ExecutionSession::IL_updateCandidatesFor(
2141 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2142 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2143 return Candidates.forEachWithRemoval(
2144 Body: [&](const SymbolStringPtr &Name,
2145 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2146 /// Search for the symbol. If not found then continue without
2147 /// removal.
2148 auto SymI = JD.Symbols.find(Val: Name);
2149 if (SymI == JD.Symbols.end())
2150 return false;
2151
2152 // If this is a non-exported symbol and we're matching exported
2153 // symbols only then remove this symbol from the candidates list.
2154 //
2155 // If we're tracking non-candidates then add this to the non-candidate
2156 // list.
2157 if (!SymI->second.getFlags().isExported() &&
2158 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2159 if (NonCandidates)
2160 NonCandidates->add(Name, Flags: SymLookupFlags);
2161 return true;
2162 }
2163
2164 // If we match against a materialization-side-effects only symbol
2165 // then make sure it is weakly-referenced. Otherwise bail out with
2166 // an error.
2167 // FIXME: Use a "materialization-side-effects-only symbols must be
2168 // weakly referenced" specific error here to reduce confusion.
2169 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2170 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2171 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2172 Args: SymbolNameVector({Name}));
2173
2174 // If we matched against this symbol but it is in the error state
2175 // then bail out and treat it as a failure to materialize.
2176 if (SymI->second.getFlags().hasError()) {
2177 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2178 (*FailedSymbolsMap)[&JD] = {Name};
2179 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2180 Args: std::move(FailedSymbolsMap));
2181 }
2182
2183 // Otherwise this is a match. Remove it from the candidate set.
2184 return true;
2185 });
2186}
2187
2188void ExecutionSession::OL_resumeLookupAfterGeneration(
2189 InProgressLookupState &IPLS) {
2190
2191 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2192 "Should not be called for not-in-generator lookups");
2193 IPLS.GenState = InProgressLookupState::NotInGenerator;
2194
2195 LookupState LS;
2196
2197 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2198 IPLS.CurDefGeneratorStack.pop_back();
2199 std::lock_guard<std::mutex> Lock(DG->M);
2200
2201 // If there are no pending lookups then mark the generator as free and
2202 // return.
2203 if (DG->PendingLookups.empty()) {
2204 DG->InUse = false;
2205 return;
2206 }
2207
2208 // Otherwise resume the next lookup.
2209 LS = std::move(DG->PendingLookups.front());
2210 DG->PendingLookups.pop_front();
2211 }
2212
2213 if (LS.IPLS) {
2214 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2215 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2216 }
2217}
2218
2219void ExecutionSession::OL_applyQueryPhase1(
2220 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2221
2222 LLVM_DEBUG({
2223 dbgs() << "Entering OL_applyQueryPhase1:\n"
2224 << " Lookup kind: " << IPLS->K << "\n"
2225 << " Search order: " << IPLS->SearchOrder
2226 << ", Current index = " << IPLS->CurSearchOrderIndex
2227 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2228 << " Lookup set: " << IPLS->LookupSet << "\n"
2229 << " Definition generator candidates: "
2230 << IPLS->DefGeneratorCandidates << "\n"
2231 << " Definition generator non-candidates: "
2232 << IPLS->DefGeneratorNonCandidates << "\n";
2233 });
2234
2235 if (IPLS->GenState == InProgressLookupState::InGenerator)
2236 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2237
2238 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2239 "Lookup should not be in InGenerator state here");
2240
2241 // FIXME: We should attach the query as we go: This provides a result in a
2242 // single pass in the common case where all symbols have already reached the
2243 // required state. The query could be detached again in the 'fail' method on
2244 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2245
2246 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2247
2248 // If we've been handed an error or received one back from a generator then
2249 // fail the query. We don't need to unlink: At this stage the query hasn't
2250 // actually been lodged.
2251 if (Err)
2252 return IPLS->fail(Err: std::move(Err));
2253
2254 // Get the next JITDylib and lookup flags.
2255 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2256 auto &JD = *KV.first;
2257 auto JDLookupFlags = KV.second;
2258
2259 LLVM_DEBUG({
2260 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2261 << ") with lookup set " << IPLS->LookupSet << ":\n";
2262 });
2263
2264 // If we've just reached a new JITDylib then perform some setup.
2265 if (IPLS->NewJITDylib) {
2266 // Add any non-candidates from the last JITDylib (if any) back on to the
2267 // list of definition candidates for this JITDylib, reset definition
2268 // non-candidates to the empty set.
2269 SymbolLookupSet Tmp;
2270 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2271 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2272
2273 LLVM_DEBUG({
2274 dbgs() << " First time visiting " << JD.getName()
2275 << ", resetting candidate sets and building generator stack\n";
2276 });
2277
2278 // Build the definition generator stack for this JITDylib.
2279 runSessionLocked(F: [&] {
2280 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2281 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2282 R: reverse(C&: JD.DefGenerators));
2283 });
2284
2285 // Flag that we've done our initialization.
2286 IPLS->NewJITDylib = false;
2287 }
2288
2289 // Remove any generation candidates that are already defined (and match) in
2290 // this JITDylib.
2291 runSessionLocked(F: [&] {
2292 // Update the list of candidates (and non-candidates) for definition
2293 // generation.
2294 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2295 Err = IL_updateCandidatesFor(
2296 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2297 NonCandidates: JD.DefGenerators.empty() ? nullptr
2298 : &IPLS->DefGeneratorNonCandidates);
2299 LLVM_DEBUG({
2300 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2301 << "\n";
2302 });
2303
2304 // If this lookup was resumed after auto-suspension but all candidates
2305 // have already been generated (by some previous call to the generator)
2306 // treat the lookup as if it had completed generation.
2307 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2308 IPLS->DefGeneratorCandidates.empty())
2309 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2310 });
2311
2312 // If we encountered an error while filtering generation candidates then
2313 // bail out.
2314 if (Err)
2315 return IPLS->fail(Err: std::move(Err));
2316
2317 /// Apply any definition generators on the stack.
2318 LLVM_DEBUG({
2319 if (IPLS->CurDefGeneratorStack.empty())
2320 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2321 else if (IPLS->DefGeneratorCandidates.empty())
2322 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2323 else
2324 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2325 << " remaining generators for "
2326 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2327 });
2328 while (!IPLS->CurDefGeneratorStack.empty() &&
2329 !IPLS->DefGeneratorCandidates.empty()) {
2330 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2331
2332 if (!DG)
2333 return IPLS->fail(Err: make_error<StringError>(
2334 Args: "DefinitionGenerator removed while lookup in progress",
2335 Args: inconvertibleErrorCode()));
2336
2337 // At this point the lookup is in either the NotInGenerator state, or in
2338 // the ResumedForGenerator state.
2339 // If this lookup is in the NotInGenerator state then check whether the
2340 // generator is in use. If the generator is not in use then move the
2341 // lookup to the InGenerator state and continue. If the generator is
2342 // already in use then just add this lookup to the pending lookups list
2343 // and bail out.
2344 // If this lookup is in the ResumedForGenerator state then just move it
2345 // to InGenerator and continue.
2346 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2347 std::lock_guard<std::mutex> Lock(DG->M);
2348 if (DG->InUse) {
2349 DG->PendingLookups.push_back(x: std::move(IPLS));
2350 return;
2351 }
2352 DG->InUse = true;
2353 }
2354
2355 IPLS->GenState = InProgressLookupState::InGenerator;
2356
2357 auto K = IPLS->K;
2358 auto &LookupSet = IPLS->DefGeneratorCandidates;
2359
2360 // Run the generator. If the generator takes ownership of QA then this
2361 // will break the loop.
2362 {
2363 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2364 LookupState LS(std::move(IPLS));
2365 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2366 IPLS = std::move(LS.IPLS);
2367 }
2368
2369 // If the lookup returned then pop the generator stack and unblock the
2370 // next lookup on this generator (if any).
2371 if (IPLS)
2372 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2373
2374 // If there was an error then fail the query.
2375 if (Err) {
2376 LLVM_DEBUG({
2377 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2378 });
2379 assert(IPLS && "LS cannot be retained if error is returned");
2380 return IPLS->fail(Err: std::move(Err));
2381 }
2382
2383 // Otherwise if QA was captured then break the loop.
2384 if (!IPLS) {
2385 LLVM_DEBUG(
2386 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2387 return;
2388 }
2389
2390 // Otherwise if we're continuing around the loop then update candidates
2391 // for the next round.
2392 runSessionLocked(F: [&] {
2393 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2394 Err = IL_updateCandidatesFor(
2395 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2396 NonCandidates: JD.DefGenerators.empty() ? nullptr
2397 : &IPLS->DefGeneratorNonCandidates);
2398 });
2399
2400 // If updating candidates failed then fail the query.
2401 if (Err) {
2402 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2403 return IPLS->fail(Err: std::move(Err));
2404 }
2405 }
2406
2407 if (IPLS->DefGeneratorCandidates.empty() &&
2408 IPLS->DefGeneratorNonCandidates.empty()) {
2409 // Early out if there are no remaining symbols.
2410 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2411 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2412 break;
2413 } else {
2414 // If we get here then we've moved on to the next JITDylib with candidates
2415 // remaining.
2416 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2417 ++IPLS->CurSearchOrderIndex;
2418 IPLS->NewJITDylib = true;
2419 }
2420 }
2421
2422 // Remove any weakly referenced candidates that could not be found/generated.
2423 IPLS->DefGeneratorCandidates.remove_if(
2424 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2425 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2426 });
2427
2428 // If we get here then we've finished searching all JITDylibs.
2429 // If we matched all symbols then move to phase 2, otherwise fail the query
2430 // with a SymbolsNotFound error.
2431 if (IPLS->DefGeneratorCandidates.empty()) {
2432 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2433 IPLS->complete(IPLS: std::move(IPLS));
2434 } else {
2435 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2436 IPLS->fail(Err: make_error<SymbolsNotFound>(
2437 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2438 }
2439}
2440
2441void ExecutionSession::OL_completeLookup(
2442 std::unique_ptr<InProgressLookupState> IPLS,
2443 std::shared_ptr<AsynchronousSymbolQuery> Q,
2444 RegisterDependenciesFunction RegisterDependencies) {
2445
2446 LLVM_DEBUG({
2447 dbgs() << "Entering OL_completeLookup:\n"
2448 << " Lookup kind: " << IPLS->K << "\n"
2449 << " Search order: " << IPLS->SearchOrder
2450 << ", Current index = " << IPLS->CurSearchOrderIndex
2451 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2452 << " Lookup set: " << IPLS->LookupSet << "\n"
2453 << " Definition generator candidates: "
2454 << IPLS->DefGeneratorCandidates << "\n"
2455 << " Definition generator non-candidates: "
2456 << IPLS->DefGeneratorNonCandidates << "\n";
2457 });
2458
2459 bool QueryComplete = false;
2460 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2461
2462 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2463 for (auto &KV : IPLS->SearchOrder) {
2464 auto &JD = *KV.first;
2465 auto JDLookupFlags = KV.second;
2466 LLVM_DEBUG({
2467 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2468 << ") with lookup set " << IPLS->LookupSet << ":\n";
2469 });
2470
2471 auto Err = IPLS->LookupSet.forEachWithRemoval(
2472 Body: [&](const SymbolStringPtr &Name,
2473 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2474 LLVM_DEBUG({
2475 dbgs() << " Attempting to match \"" << Name << "\" ("
2476 << SymLookupFlags << ")... ";
2477 });
2478
2479 /// Search for the symbol. If not found then continue without
2480 /// removal.
2481 auto SymI = JD.Symbols.find(Val: Name);
2482 if (SymI == JD.Symbols.end()) {
2483 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2484 return false;
2485 }
2486
2487 // If this is a non-exported symbol and we're matching exported
2488 // symbols only then skip this symbol without removal.
2489 if (!SymI->second.getFlags().isExported() &&
2490 JDLookupFlags ==
2491 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2492 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2493 return false;
2494 }
2495
2496 // If we match against a materialization-side-effects only symbol
2497 // then make sure it is weakly-referenced. Otherwise bail out with
2498 // an error.
2499 // FIXME: Use a "materialization-side-effects-only symbols must be
2500 // weakly referenced" specific error here to reduce confusion.
2501 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2502 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2503 LLVM_DEBUG({
2504 dbgs() << "error: "
2505 "required, but symbol is has-side-effects-only\n";
2506 });
2507 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2508 Args: SymbolNameVector({Name}));
2509 }
2510
2511 // If we matched against this symbol but it is in the error state
2512 // then bail out and treat it as a failure to materialize.
2513 if (SymI->second.getFlags().hasError()) {
2514 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2515 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2516 (*FailedSymbolsMap)[&JD] = {Name};
2517 return make_error<FailedToMaterialize>(
2518 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2519 }
2520
2521 // Otherwise this is a match.
2522
2523 // If this symbol is already in the required state then notify the
2524 // query, remove the symbol and continue.
2525 if (SymI->second.getState() >= Q->getRequiredState()) {
2526 LLVM_DEBUG(dbgs()
2527 << "matched, symbol already in required state\n");
2528 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2529
2530 // If this symbol is in anything other than the Ready state then
2531 // we need to track the dependence.
2532 if (SymI->second.getState() != SymbolState::Ready)
2533 Q->addQueryDependence(JD, Name);
2534
2535 return true;
2536 }
2537
2538 // Otherwise this symbol does not yet meet the required state. Check
2539 // whether it has a materializer attached, and if so prepare to run
2540 // it.
2541 if (SymI->second.hasMaterializerAttached()) {
2542 assert(SymI->second.getAddress() == ExecutorAddr() &&
2543 "Symbol not resolved but already has address?");
2544 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2545 assert(UMII != JD.UnmaterializedInfos.end() &&
2546 "Lazy symbol should have UnmaterializedInfo");
2547
2548 auto UMI = UMII->second;
2549 assert(UMI->MU && "Materializer should not be null");
2550 assert(UMI->RT && "Tracker should not be null");
2551 LLVM_DEBUG({
2552 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2553 << " (" << UMI->MU->getName() << ")\n";
2554 });
2555
2556 // Move all symbols associated with this MaterializationUnit into
2557 // materializing state.
2558 for (auto &KV : UMI->MU->getSymbols()) {
2559 auto SymK = JD.Symbols.find(Val: KV.first);
2560 assert(SymK != JD.Symbols.end() &&
2561 "No entry for symbol covered by MaterializationUnit");
2562 SymK->second.setMaterializerAttached(false);
2563 SymK->second.setState(SymbolState::Materializing);
2564 JD.UnmaterializedInfos.erase(Val: KV.first);
2565 }
2566
2567 // Add MU to the list of MaterializationUnits to be materialized.
2568 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2569 } else
2570 LLVM_DEBUG(dbgs() << "matched, registering query");
2571
2572 // Add the query to the PendingQueries list and continue, deleting
2573 // the element from the lookup set.
2574 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2575 SymI->second.getState() != SymbolState::Ready &&
2576 "By this line the symbol should be materializing");
2577 auto &MI = JD.MaterializingInfos[Name];
2578 MI.addQuery(Q);
2579 Q->addQueryDependence(JD, Name);
2580
2581 return true;
2582 });
2583
2584 JD.shrinkMaterializationInfoMemory();
2585
2586 // Handle failure.
2587 if (Err) {
2588
2589 LLVM_DEBUG({
2590 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2591 });
2592
2593 // Detach the query.
2594 Q->detach();
2595
2596 // Replace the MUs.
2597 for (auto &KV : CollectedUMIs) {
2598 auto &JD = *KV.first;
2599 for (auto &UMI : KV.second)
2600 for (auto &KV2 : UMI->MU->getSymbols()) {
2601 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2602 "Unexpected materializer in map");
2603 auto SymI = JD.Symbols.find(Val: KV2.first);
2604 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2605 assert(SymI->second.getState() == SymbolState::Materializing &&
2606 "Can not replace symbol that is not materializing");
2607 assert(!SymI->second.hasMaterializerAttached() &&
2608 "MaterializerAttached flag should not be set");
2609 SymI->second.setMaterializerAttached(true);
2610 JD.UnmaterializedInfos[KV2.first] = UMI;
2611 }
2612 }
2613
2614 return Err;
2615 }
2616 }
2617
2618 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2619 IPLS->LookupSet.forEachWithRemoval(
2620 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2621 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2622 Q->dropSymbol(Name);
2623 return true;
2624 } else
2625 return false;
2626 });
2627
2628 if (!IPLS->LookupSet.empty()) {
2629 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2630 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2631 Args: IPLS->LookupSet.getSymbolNames());
2632 }
2633
2634 // Record whether the query completed.
2635 QueryComplete = Q->isComplete();
2636
2637 LLVM_DEBUG({
2638 dbgs() << "Query successfully "
2639 << (QueryComplete ? "completed" : "lodged") << "\n";
2640 });
2641
2642 // Move the collected MUs to the OutstandingMUs list.
2643 if (!CollectedUMIs.empty()) {
2644 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2645
2646 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2647 for (auto &KV : CollectedUMIs) {
2648 LLVM_DEBUG({
2649 auto &JD = *KV.first;
2650 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2651 << " MUs.\n";
2652 });
2653 for (auto &UMI : KV.second) {
2654 auto MR = createMaterializationResponsibility(
2655 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2656 InitSymbol: std::move(UMI->MU->InitSymbol));
2657 OutstandingMUs.push_back(
2658 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2659 }
2660 }
2661 } else
2662 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2663
2664 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2665 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2666 RegisterDependencies(Q->QueryRegistrations);
2667 } else
2668 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2669
2670 return Error::success();
2671 });
2672
2673 if (LodgingErr) {
2674 LLVM_DEBUG(dbgs() << "Failing query\n");
2675 Q->detach();
2676 Q->handleFailed(Err: std::move(LodgingErr));
2677 return;
2678 }
2679
2680 if (QueryComplete) {
2681 LLVM_DEBUG(dbgs() << "Completing query\n");
2682 Q->handleComplete(ES&: *this);
2683 }
2684
2685 dispatchOutstandingMUs();
2686}
2687
2688void ExecutionSession::OL_completeLookupFlags(
2689 std::unique_ptr<InProgressLookupState> IPLS,
2690 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2691
2692 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2693 LLVM_DEBUG({
2694 dbgs() << "Entering OL_completeLookupFlags:\n"
2695 << " Lookup kind: " << IPLS->K << "\n"
2696 << " Search order: " << IPLS->SearchOrder
2697 << ", Current index = " << IPLS->CurSearchOrderIndex
2698 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2699 << " Lookup set: " << IPLS->LookupSet << "\n"
2700 << " Definition generator candidates: "
2701 << IPLS->DefGeneratorCandidates << "\n"
2702 << " Definition generator non-candidates: "
2703 << IPLS->DefGeneratorNonCandidates << "\n";
2704 });
2705
2706 SymbolFlagsMap Result;
2707
2708 // Attempt to find flags for each symbol.
2709 for (auto &KV : IPLS->SearchOrder) {
2710 auto &JD = *KV.first;
2711 auto JDLookupFlags = KV.second;
2712 LLVM_DEBUG({
2713 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2714 << ") with lookup set " << IPLS->LookupSet << ":\n";
2715 });
2716
2717 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2718 SymbolLookupFlags SymLookupFlags) {
2719 LLVM_DEBUG({
2720 dbgs() << " Attempting to match \"" << Name << "\" ("
2721 << SymLookupFlags << ")... ";
2722 });
2723
2724 // Search for the symbol. If not found then continue without removing
2725 // from the lookup set.
2726 auto SymI = JD.Symbols.find(Val: Name);
2727 if (SymI == JD.Symbols.end()) {
2728 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2729 return false;
2730 }
2731
2732 // If this is a non-exported symbol then it doesn't match. Skip it.
2733 if (!SymI->second.getFlags().isExported() &&
2734 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2735 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2736 return false;
2737 }
2738
2739 LLVM_DEBUG({
2740 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2741 << "\n";
2742 });
2743 Result[Name] = SymI->second.getFlags();
2744 return true;
2745 });
2746 }
2747
2748 // Remove any weakly referenced symbols that haven't been resolved.
2749 IPLS->LookupSet.remove_if(
2750 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2751 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2752 });
2753
2754 if (!IPLS->LookupSet.empty()) {
2755 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2756 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2757 Args: IPLS->LookupSet.getSymbolNames());
2758 }
2759
2760 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2761 return Result;
2762 });
2763
2764 // Run the callback on the result.
2765 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2766 OnComplete(std::move(Result));
2767}
2768
2769void ExecutionSession::OL_destroyMaterializationResponsibility(
2770 MaterializationResponsibility &MR) {
2771
2772 assert(MR.SymbolFlags.empty() &&
2773 "All symbols should have been explicitly materialized or failed");
2774 MR.JD.unlinkMaterializationResponsibility(MR);
2775}
2776
2777SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2778 const MaterializationResponsibility &MR) {
2779 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2780}
2781
2782Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2783 const SymbolMap &Symbols) {
2784 LLVM_DEBUG({
2785 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2786 });
2787#ifndef NDEBUG
2788 for (auto &KV : Symbols) {
2789 auto I = MR.SymbolFlags.find(KV.first);
2790 assert(I != MR.SymbolFlags.end() &&
2791 "Resolving symbol outside this responsibility set");
2792 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2793 "Can't resolve materialization-side-effects-only symbol");
2794 if (I->second & JITSymbolFlags::Common) {
2795 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2796 assert((KV.second.getFlags() & WeakOrCommon) &&
2797 "Common symbols must be resolved as common or weak");
2798 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2799 (I->second & ~JITSymbolFlags::Common) &&
2800 "Resolving symbol with incorrect flags");
2801 } else
2802 assert(KV.second.getFlags() == I->second &&
2803 "Resolving symbol with incorrect flags");
2804 }
2805#endif
2806
2807 return MR.JD.resolve(MR, Resolved: Symbols);
2808}
2809
2810WaitingOnGraph::ExternalState
2811ExecutionSession::IL_getSymbolState(JITDylib *JD,
2812 NonOwningSymbolStringPtr Name) {
2813 if (JD->State != JITDylib::Open)
2814 return WaitingOnGraph::ExternalState::Failed;
2815
2816 auto I = JD->Symbols.find_as(Val: Name);
2817
2818 // FIXME: Can we eliminate this possibility if we support query binding?
2819 if (I == JD->Symbols.end())
2820 return WaitingOnGraph::ExternalState::Failed;
2821
2822 if (I->second.getFlags().hasError())
2823 return WaitingOnGraph::ExternalState::Failed;
2824
2825 if (I->second.getState() == SymbolState::Ready)
2826 return WaitingOnGraph::ExternalState::Ready;
2827
2828 return WaitingOnGraph::ExternalState::None;
2829}
2830
2831template <typename UpdateSymbolFn, typename UpdateQueryFn>
2832void ExecutionSession::IL_collectQueries(
2833 JITDylib::AsynchronousSymbolQuerySet &Qs,
2834 WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
2835 UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
2836
2837 for (auto &[JD, Symbols] : QualifiedSymbols) {
2838 // IL_emit and JITDylib removal are synchronized by the session lock.
2839 // Since JITDylib removal removes any contained nodes from the
2840 // WaitingOnGraph, we should be able to assert that all nodes in the
2841 // WaitingOnGraph have not been removed.
2842 assert(JD->State == JITDylib::Open &&
2843 "WaitingOnGraph includes definition in defunct JITDylib");
2844 for (auto &Symbol : Symbols) {
2845 // Update symbol table.
2846 auto I = JD->Symbols.find_as(Val: Symbol);
2847 assert(I != JD->Symbols.end() &&
2848 "Failed Symbol missing from JD symbol table");
2849 auto &Entry = I->second;
2850 UpdateSymbol(Entry);
2851
2852 // Collect queries.
2853 auto J = JD->MaterializingInfos.find_as(Val: Symbol);
2854 if (J != JD->MaterializingInfos.end()) {
2855 for (auto &Q : J->second.takeAllPendingQueries()) {
2856 UpdateQuery(*Q, *JD, Symbol, Entry);
2857 Qs.insert(x: std::move(Q));
2858 }
2859 JD->MaterializingInfos.erase(I: J);
2860 }
2861 }
2862 }
2863}
2864
2865Expected<ExecutionSession::EmitQueries>
2866ExecutionSession::IL_emit(MaterializationResponsibility &MR,
2867 WaitingOnGraph::SimplifyResult SR) {
2868
2869 if (MR.RT->isDefunct())
2870 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
2871
2872 auto &TargetJD = MR.getTargetJITDylib();
2873 if (TargetJD.State != JITDylib::Open)
2874 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
2875 " is defunct",
2876 Args: inconvertibleErrorCode());
2877
2878#ifdef EXPENSIVE_CHECKS
2879 verifySessionState("entering ExecutionSession::IL_emit");
2880#endif
2881
2882 auto ER = G.emit(SR: std::move(SR),
2883 GetExternalState: [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
2884 return IL_getSymbolState(JD, Name);
2885 });
2886
2887 EmitQueries EQ;
2888
2889 // Handle failed queries.
2890 for (auto &SN : ER.Failed)
2891 IL_collectQueries(
2892 Qs&: EQ.Failed, QualifiedSymbols&: SN->defs(),
2893 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) {
2894 E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
2895 },
2896 UpdateQuery: [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
2897 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2898 auto &FS = EQ.FailedSymsForQuery[&Q];
2899 if (!FS)
2900 FS = std::make_shared<SymbolDependenceMap>();
2901 (*FS)[&JD].insert(V: SymbolStringPtr(Name));
2902 });
2903
2904 for (auto &FQ : EQ.Failed)
2905 FQ->detach();
2906
2907 for (auto &SN : ER.Ready)
2908 IL_collectQueries(
2909 Qs&: EQ.Completed, QualifiedSymbols&: SN->defs(),
2910 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
2911 UpdateQuery: [](AsynchronousSymbolQuery &Q, JITDylib &JD,
2912 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2913 Q.notifySymbolMetRequiredState(Name: SymbolStringPtr(Name), Sym: E.getSymbol());
2914 });
2915
2916 // std::erase_if is not available in C++17, and llvm::erase_if does not work
2917 // here.
2918 for (auto it = EQ.Completed.begin(), end = EQ.Completed.end(); it != end;) {
2919 if ((*it)->isComplete()) {
2920 ++it;
2921 } else {
2922 it = EQ.Completed.erase(position: it);
2923 }
2924 }
2925
2926#ifdef EXPENSIVE_CHECKS
2927 verifySessionState("exiting ExecutionSession::IL_emit");
2928#endif
2929
2930 return std::move(EQ);
2931}
2932
2933Error ExecutionSession::OL_notifyEmitted(
2934 MaterializationResponsibility &MR,
2935 ArrayRef<SymbolDependenceGroup> DepGroups) {
2936 LLVM_DEBUG({
2937 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
2938 << "\n";
2939 if (!DepGroups.empty()) {
2940 dbgs() << " Initial dependencies:\n";
2941 for (auto &SDG : DepGroups) {
2942 dbgs() << " Symbols: " << SDG.Symbols
2943 << ", Dependencies: " << SDG.Dependencies << "\n";
2944 }
2945 }
2946 });
2947
2948#ifndef NDEBUG
2949 SymbolNameSet Visited;
2950 for (auto &DG : DepGroups) {
2951 for (auto &Sym : DG.Symbols) {
2952 assert(MR.SymbolFlags.count(Sym) &&
2953 "DG contains dependence for symbol outside this MR");
2954 assert(Visited.insert(Sym).second &&
2955 "DG contains duplicate entries for Name");
2956 }
2957 }
2958#endif // NDEBUG
2959
2960 std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
2961 WaitingOnGraph::ContainerElementsMap Residual;
2962 {
2963 auto &JDResidual = Residual[&MR.getTargetJITDylib()];
2964 for (auto &[Name, Flags] : MR.getSymbols())
2965 JDResidual.insert(V: NonOwningSymbolStringPtr(Name));
2966
2967 for (auto &SDG : DepGroups) {
2968 WaitingOnGraph::ContainerElementsMap Defs;
2969 assert(!SDG.Symbols.empty());
2970 auto &JDDefs = Defs[&MR.getTargetJITDylib()];
2971 for (auto &Def : SDG.Symbols) {
2972 JDDefs.insert(V: NonOwningSymbolStringPtr(Def));
2973 JDResidual.erase(V: NonOwningSymbolStringPtr(Def));
2974 }
2975 WaitingOnGraph::ContainerElementsMap Deps;
2976 if (!SDG.Dependencies.empty()) {
2977 for (auto &[JD, Syms] : SDG.Dependencies) {
2978 auto &JDDeps = Deps[JD];
2979 for (auto &Dep : Syms)
2980 JDDeps.insert(V: NonOwningSymbolStringPtr(Dep));
2981 }
2982 }
2983 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2984 args: std::move(Defs), args: std::move(Deps)));
2985 }
2986 if (!JDResidual.empty())
2987 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2988 args: std::move(Residual), args: WaitingOnGraph::ContainerElementsMap()));
2989 }
2990
2991 auto SR = WaitingOnGraph::simplify(SNs: std::move(SNs));
2992
2993 LLVM_DEBUG({
2994 dbgs() << " Simplified dependencies:\n";
2995 for (auto &SN : SR.superNodes()) {
2996
2997 auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
2998 std::vector<JITDylib *> JDs;
2999 for (auto &[JD, _] : C)
3000 JDs.push_back(JD);
3001 llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
3002 return LHS->getName() < RHS->getName();
3003 });
3004 return JDs;
3005 };
3006
3007 auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
3008 std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
3009 llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
3010 const NonOwningSymbolStringPtr &RHS) {
3011 return *LHS < *RHS;
3012 });
3013 return Names;
3014 };
3015
3016 dbgs() << " Defs: {";
3017 for (auto *JD : SortedLibs(SN->defs())) {
3018 dbgs() << " (" << JD->getName() << ", [";
3019 for (auto &Sym : SortedNames(SN->defs()[JD]))
3020 dbgs() << " " << Sym;
3021 dbgs() << " ])";
3022 }
3023 dbgs() << " }, Deps: {";
3024 for (auto *JD : SortedLibs(SN->deps())) {
3025 dbgs() << " (" << JD->getName() << ", [";
3026 for (auto &Sym : SortedNames(SN->deps()[JD]))
3027 dbgs() << " " << Sym;
3028 dbgs() << " ])";
3029 }
3030 dbgs() << " }\n";
3031 }
3032 });
3033 auto EmitQueries =
3034 runSessionLocked(F: [&]() { return IL_emit(MR, SR: std::move(SR)); });
3035
3036 // On error bail out.
3037 if (!EmitQueries)
3038 return EmitQueries.takeError();
3039
3040 // Otherwise notify failed queries, and any updated queries that have been
3041 // completed.
3042
3043 // FIXME: Get rid of error return from notifyEmitted.
3044 SymbolDependenceMap BadDeps;
3045 {
3046 for (auto &FQ : EmitQueries->Failed) {
3047 FQ->detach();
3048 assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
3049 "Missing failed symbols for query");
3050 auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
3051 for (auto &[JD, Syms] : *FailedSyms) {
3052 auto &BadDepsForJD = BadDeps[JD];
3053 for (auto &Sym : Syms)
3054 BadDepsForJD.insert(V: Sym);
3055 }
3056 FQ->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
3057 Args: std::move(FailedSyms)));
3058 }
3059 }
3060
3061 for (auto &UQ : EmitQueries->Completed)
3062 UQ->handleComplete(ES&: *this);
3063
3064 // If there are any bad dependencies then return an error.
3065 if (!BadDeps.empty()) {
3066 SymbolNameSet BadNames;
3067 // Note: The name set calculated here is bogus: it includes all symbols in
3068 // the MR, not just the ones that failed. We want to remove the error
3069 // return path from notifyEmitted anyway, so this is just a brief
3070 // placeholder to maintain (roughly) the current error behavior.
3071 for (auto &[Name, Flags] : MR.getSymbols())
3072 BadNames.insert(V: Name);
3073 MR.SymbolFlags.clear();
3074 return make_error<UnsatisfiedSymbolDependencies>(
3075 Args: getSymbolStringPool(), Args: &MR.getTargetJITDylib(), Args: std::move(BadNames),
3076 Args: std::move(BadDeps), Args: "dependencies removed or in error state");
3077 }
3078
3079 MR.SymbolFlags.clear();
3080 return Error::success();
3081}
3082
3083Error ExecutionSession::OL_defineMaterializing(
3084 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3085
3086 LLVM_DEBUG({
3087 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3088 << NewSymbolFlags << "\n";
3089 });
3090 if (auto AcceptedDefs =
3091 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3092 // Add all newly accepted symbols to this responsibility object.
3093 for (auto &KV : *AcceptedDefs)
3094 MR.SymbolFlags.insert(KV);
3095 return Error::success();
3096 } else
3097 return AcceptedDefs.takeError();
3098}
3099
3100std::pair<JITDylib::AsynchronousSymbolQuerySet,
3101 std::shared_ptr<SymbolDependenceMap>>
3102ExecutionSession::IL_failSymbols(JITDylib &JD,
3103 const SymbolNameVector &SymbolsToFail) {
3104
3105#ifdef EXPENSIVE_CHECKS
3106 verifySessionState("entering ExecutionSession::IL_failSymbols");
3107#endif
3108
3109 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3110 auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
3111 auto I = FailJD->Symbols.find_as(Val: FailSym);
3112 assert(I != FailJD->Symbols.end());
3113 I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
3114 auto J = FailJD->MaterializingInfos.find_as(Val: FailSym);
3115 if (J != FailJD->MaterializingInfos.end()) {
3116 for (auto &Q : J->second.takeAllPendingQueries())
3117 FailedQueries.insert(x: std::move(Q));
3118 FailJD->MaterializingInfos.erase(I: J);
3119 }
3120 };
3121
3122 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3123
3124 {
3125 auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
3126 for (auto &Sym : SymbolsToFail) {
3127 FailedSymsForJD.insert(V: Sym);
3128 Fail(&JD, NonOwningSymbolStringPtr(Sym));
3129 }
3130 }
3131
3132 WaitingOnGraph::ContainerElementsMap ToFail;
3133 auto &JDToFail = ToFail[&JD];
3134 for (auto &Sym : SymbolsToFail)
3135 JDToFail.insert(V: NonOwningSymbolStringPtr(Sym));
3136
3137 auto FailedSNs = G.fail(Failed: ToFail);
3138
3139 for (auto &SN : FailedSNs) {
3140 for (auto &[FailJD, Defs] : SN->defs()) {
3141 auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
3142 for (auto &Def : Defs) {
3143 FailedSymsForFailJD.insert(V: SymbolStringPtr(Def));
3144 Fail(FailJD, Def);
3145 }
3146 }
3147 }
3148
3149 // Detach all failed queries.
3150 for (auto &Q : FailedQueries)
3151 Q->detach();
3152
3153#ifdef EXPENSIVE_CHECKS
3154 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3155#endif
3156
3157 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3158}
3159
3160void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3161
3162 LLVM_DEBUG({
3163 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3164 << MR.SymbolFlags << "\n";
3165 });
3166
3167 if (MR.SymbolFlags.empty())
3168 return;
3169
3170 SymbolNameVector SymbolsToFail;
3171 for (auto &[Name, Flags] : MR.SymbolFlags)
3172 SymbolsToFail.push_back(x: Name);
3173 MR.SymbolFlags.clear();
3174
3175 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3176 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3177
3178 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3179 // If the tracker is defunct then there's nothing to do here.
3180 if (MR.RT->isDefunct())
3181 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3182 std::shared_ptr<SymbolDependenceMap>>();
3183 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3184 });
3185
3186 for (auto &Q : FailedQueries) {
3187 Q->detach();
3188 Q->handleFailed(
3189 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3190 }
3191}
3192
3193Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3194 std::unique_ptr<MaterializationUnit> MU) {
3195 for (auto &KV : MU->getSymbols()) {
3196 assert(MR.SymbolFlags.count(KV.first) &&
3197 "Replacing definition outside this responsibility set");
3198 MR.SymbolFlags.erase(Val: KV.first);
3199 }
3200
3201 if (MU->getInitializerSymbol() == MR.InitSymbol)
3202 MR.InitSymbol = nullptr;
3203
3204 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3205 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3206 << "\n";
3207 }););
3208
3209 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3210}
3211
3212Expected<std::unique_ptr<MaterializationResponsibility>>
3213ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3214 const SymbolNameSet &Symbols) {
3215
3216 SymbolStringPtr DelegatedInitSymbol;
3217 SymbolFlagsMap DelegatedFlags;
3218
3219 for (auto &Name : Symbols) {
3220 auto I = MR.SymbolFlags.find(Val: Name);
3221 assert(I != MR.SymbolFlags.end() &&
3222 "Symbol is not tracked by this MaterializationResponsibility "
3223 "instance");
3224
3225 DelegatedFlags[Name] = std::move(I->second);
3226 if (Name == MR.InitSymbol)
3227 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3228
3229 MR.SymbolFlags.erase(I);
3230 }
3231
3232 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3233 InitSymbol: std::move(DelegatedInitSymbol));
3234}
3235
3236#ifndef NDEBUG
3237void ExecutionSession::dumpDispatchInfo(Task &T) {
3238 runSessionLocked([&]() {
3239 dbgs() << "Dispatching: ";
3240 T.printDescription(dbgs());
3241 dbgs() << "\n";
3242 });
3243}
3244#endif // NDEBUG
3245
3246} // End namespace orc.
3247} // End namespace llvm.
3248