1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
15#include "llvm/Support/FormatVariadic.h"
16#include "llvm/Support/MSVCErrorWorkarounds.h"
17#include "llvm/Support/raw_ostream.h"
18
19#include <condition_variable>
20#include <future>
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25namespace llvm {
26namespace orc {
27
28char ResourceTrackerDefunct::ID = 0;
29char JITDylibDefunct::ID = 0;
30char FailedToMaterialize::ID = 0;
31char SymbolsNotFound::ID = 0;
32char SymbolsCouldNotBeRemoved::ID = 0;
33char MissingSymbolDefinitions::ID = 0;
34char UnexpectedSymbolDefinitions::ID = 0;
35char UnsatisfiedSymbolDependencies::ID = 0;
36char MaterializationTask::ID = 0;
37char LookupTask::ID = 0;
38
39RegisterDependenciesFunction NoDependenciesToRegister =
40 RegisterDependenciesFunction();
41
42void MaterializationUnit::anchor() {}
43
44ResourceTracker::ResourceTracker(JITDylibSP JD) {
45 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
46 "JITDylib must be two byte aligned");
47 JD->Retain();
48 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
49}
50
51ResourceTracker::~ResourceTracker() {
52 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
53 getJITDylib().Release();
54}
55
56Error ResourceTracker::remove() {
57 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
58}
59
60void ResourceTracker::transferTo(ResourceTracker &DstRT) {
61 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
62}
63
64void ResourceTracker::makeDefunct() {
65 uintptr_t Val = JDAndFlag.load();
66 Val |= 0x1U;
67 JDAndFlag.store(i: Val);
68}
69
70ResourceManager::~ResourceManager() = default;
71
72ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
73 : RT(std::move(RT)) {}
74
75std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
76 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
77}
78
79void ResourceTrackerDefunct::log(raw_ostream &OS) const {
80 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
81}
82
83std::error_code JITDylibDefunct::convertToErrorCode() const {
84 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
85}
86
87void JITDylibDefunct::log(raw_ostream &OS) const {
88 OS << "JITDylib " << JD->getName() << " (" << (void *)JD.get()
89 << ") is defunct";
90}
91
92FailedToMaterialize::FailedToMaterialize(
93 std::shared_ptr<SymbolStringPool> SSP,
94 std::shared_ptr<SymbolDependenceMap> Symbols)
95 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
96 assert(this->SSP && "String pool cannot be null");
97 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
98
99 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
100 // don't have to manually retain/release.
101 for (auto &[JD, Syms] : *this->Symbols)
102 JD->Retain();
103}
104
105FailedToMaterialize::~FailedToMaterialize() {
106 for (auto &[JD, Syms] : *Symbols)
107 JD->Release();
108}
109
110std::error_code FailedToMaterialize::convertToErrorCode() const {
111 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
112}
113
114void FailedToMaterialize::log(raw_ostream &OS) const {
115 OS << "Failed to materialize symbols: " << *Symbols;
116}
117
118UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
119 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
120 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
121 std::string Explanation)
122 : SSP(std::move(SSP)), JD(std::move(JD)),
123 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
124 Explanation(std::move(Explanation)) {}
125
126std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
127 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
128}
129
130void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
131 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
132 << ", due to unsatisfied dependencies " << BadDeps;
133 if (!Explanation.empty())
134 OS << " (" << Explanation << ")";
135}
136
137SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
138 SymbolNameSet Symbols)
139 : SSP(std::move(SSP)) {
140 llvm::append_range(C&: this->Symbols, R&: Symbols);
141 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
142}
143
144SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
145 SymbolNameVector Symbols)
146 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
147 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
148}
149
150std::error_code SymbolsNotFound::convertToErrorCode() const {
151 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
152}
153
154void SymbolsNotFound::log(raw_ostream &OS) const {
155 OS << "Symbols not found: " << Symbols;
156}
157
158SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
159 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
160 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
161 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
162}
163
164std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
165 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
166}
167
168void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
169 OS << "Symbols could not be removed: " << Symbols;
170}
171
172std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
173 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
174}
175
176void MissingSymbolDefinitions::log(raw_ostream &OS) const {
177 OS << "Missing definitions in module " << ModuleName
178 << ": " << Symbols;
179}
180
181std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
182 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
183}
184
185void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
186 OS << "Unexpected definitions in module " << ModuleName
187 << ": " << Symbols;
188}
189
190void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
191 JD->getExecutionSession().lookup(
192 K: LookupKind::Static, SearchOrder: {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
193 Symbols: SymbolLookupSet(Name), RequiredState: SymbolState::Ready,
194 NotifyComplete: [OnComplete = std::move(OnComplete)
195#ifndef NDEBUG
196 ,
197 Name = this->Name // Captured for the assert below only.
198#endif // NDEBUG
199 ](Expected<SymbolMap> Result) mutable {
200 if (Result) {
201 assert(Result->size() == 1 && "Unexpected number of results");
202 assert(Result->count(Name) &&
203 "Result does not contain expected symbol");
204 OnComplete(Result->begin()->second);
205 } else
206 OnComplete(Result.takeError());
207 },
208 RegisterDependencies: NoDependenciesToRegister);
209}
210
211AsynchronousSymbolQuery::AsynchronousSymbolQuery(
212 const SymbolLookupSet &Symbols, SymbolState RequiredState,
213 SymbolsResolvedCallback NotifyComplete)
214 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
215 assert(RequiredState >= SymbolState::Resolved &&
216 "Cannot query for a symbols that have not reached the resolve state "
217 "yet");
218
219 OutstandingSymbolsCount = Symbols.size();
220
221 for (auto &[Name, Flags] : Symbols)
222 ResolvedSymbols[Name] = ExecutorSymbolDef();
223}
224
225void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
226 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
227 auto I = ResolvedSymbols.find(Val: Name);
228 assert(I != ResolvedSymbols.end() &&
229 "Resolving symbol outside the requested set");
230 assert(I->second == ExecutorSymbolDef() &&
231 "Redundantly resolving symbol Name");
232
233 // If this is a materialization-side-effects-only symbol then drop it,
234 // otherwise update its map entry with its resolved address.
235 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
236 ResolvedSymbols.erase(I);
237 else
238 I->second = std::move(Sym);
239 --OutstandingSymbolsCount;
240}
241
242void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
243 assert(OutstandingSymbolsCount == 0 &&
244 "Symbols remain, handleComplete called prematurely");
245
246 class RunQueryCompleteTask : public Task {
247 public:
248 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
249 SymbolsResolvedCallback NotifyComplete)
250 : ResolvedSymbols(std::move(ResolvedSymbols)),
251 NotifyComplete(std::move(NotifyComplete)) {}
252 void printDescription(raw_ostream &OS) override {
253 OS << "Execute query complete callback for " << ResolvedSymbols;
254 }
255 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
256
257 private:
258 SymbolMap ResolvedSymbols;
259 SymbolsResolvedCallback NotifyComplete;
260 };
261
262 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
263 args: std::move(NotifyComplete));
264 NotifyComplete = SymbolsResolvedCallback();
265 ES.dispatchTask(T: std::move(T));
266}
267
268void AsynchronousSymbolQuery::handleFailed(Error Err) {
269 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
270 OutstandingSymbolsCount == 0 &&
271 "Query should already have been abandoned");
272 NotifyComplete(std::move(Err));
273 NotifyComplete = SymbolsResolvedCallback();
274}
275
276void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
277 SymbolStringPtr Name) {
278 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
279 (void)Added;
280 assert(Added && "Duplicate dependence notification?");
281}
282
283void AsynchronousSymbolQuery::removeQueryDependence(
284 JITDylib &JD, const SymbolStringPtr &Name) {
285 auto QRI = QueryRegistrations.find(Val: &JD);
286 assert(QRI != QueryRegistrations.end() &&
287 "No dependencies registered for JD");
288 assert(QRI->second.count(Name) && "No dependency on Name in JD");
289 QRI->second.erase(V: Name);
290 if (QRI->second.empty())
291 QueryRegistrations.erase(I: QRI);
292}
293
294void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
295 auto I = ResolvedSymbols.find(Val: Name);
296 assert(I != ResolvedSymbols.end() &&
297 "Redundant removal of weakly-referenced symbol");
298 ResolvedSymbols.erase(I);
299 --OutstandingSymbolsCount;
300}
301
302void AsynchronousSymbolQuery::detach() {
303 ResolvedSymbols.clear();
304 OutstandingSymbolsCount = 0;
305 for (auto &[JD, Syms] : QueryRegistrations)
306 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
307 QueryRegistrations.clear();
308}
309
310ReExportsMaterializationUnit::ReExportsMaterializationUnit(
311 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
312 SymbolAliasMap Aliases)
313 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
314 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
315
316StringRef ReExportsMaterializationUnit::getName() const {
317 return "<Reexports>";
318}
319
320void ReExportsMaterializationUnit::materialize(
321 std::unique_ptr<MaterializationResponsibility> R) {
322
323 auto &ES = R->getTargetJITDylib().getExecutionSession();
324 JITDylib &TgtJD = R->getTargetJITDylib();
325 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
326
327 // Find the set of requested aliases and aliasees. Return any unrequested
328 // aliases back to the JITDylib so as to not prematurely materialize any
329 // aliasees.
330 auto RequestedSymbols = R->getRequestedSymbols();
331 SymbolAliasMap RequestedAliases;
332
333 for (auto &Name : RequestedSymbols) {
334 auto I = Aliases.find(Val: Name);
335 assert(I != Aliases.end() && "Symbol not found in aliases map?");
336 RequestedAliases[Name] = std::move(I->second);
337 Aliases.erase(I);
338 }
339
340 LLVM_DEBUG({
341 ES.runSessionLocked([&]() {
342 dbgs() << "materializing reexports: target = " << TgtJD.getName()
343 << ", source = " << SrcJD.getName() << " " << RequestedAliases
344 << "\n";
345 });
346 });
347
348 if (!Aliases.empty()) {
349 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
350 SourceJDLookupFlags))
351 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
352
353 if (Err) {
354 // FIXME: Should this be reported / treated as failure to materialize?
355 // Or should this be treated as a sanctioned bailing-out?
356 ES.reportError(Err: std::move(Err));
357 R->failMaterialization();
358 return;
359 }
360 }
361
362 // The OnResolveInfo struct will hold the aliases and responsibility for each
363 // query in the list.
364 struct OnResolveInfo {
365 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
366 SymbolAliasMap Aliases)
367 : R(std::move(R)), Aliases(std::move(Aliases)) {}
368
369 std::unique_ptr<MaterializationResponsibility> R;
370 SymbolAliasMap Aliases;
371 std::vector<SymbolDependenceGroup> SDGs;
372 };
373
374 // Build a list of queries to issue. In each round we build a query for the
375 // largest set of aliases that we can resolve without encountering a chain of
376 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
377 // query would be waiting on a symbol that it itself had to resolve. Creating
378 // a new query for each link in such a chain eliminates the possibility of
379 // deadlock. In practice chains are likely to be rare, and this algorithm will
380 // usually result in a single query to issue.
381
382 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
383 QueryInfos;
384 while (!RequestedAliases.empty()) {
385 SymbolNameSet ResponsibilitySymbols;
386 SymbolLookupSet QuerySymbols;
387 SymbolAliasMap QueryAliases;
388
389 // Collect as many aliases as we can without including a chain.
390 for (auto &[Alias, AliasInfo] : RequestedAliases) {
391 // Chain detected. Skip this symbol for this round.
392 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: AliasInfo.Aliasee) ||
393 RequestedAliases.count(Val: AliasInfo.Aliasee)))
394 continue;
395
396 ResponsibilitySymbols.insert(V: Alias);
397 QuerySymbols.add(Name: AliasInfo.Aliasee,
398 Flags: AliasInfo.AliasFlags.hasMaterializationSideEffectsOnly()
399 ? SymbolLookupFlags::WeaklyReferencedSymbol
400 : SymbolLookupFlags::RequiredSymbol);
401 QueryAliases[Alias] = std::move(AliasInfo);
402 }
403
404 // Remove the aliases collected this round from the RequestedAliases map.
405 for (auto &KV : QueryAliases)
406 RequestedAliases.erase(Val: KV.first);
407
408 assert(!QuerySymbols.empty() && "Alias cycle detected!");
409
410 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
411 if (!NewR) {
412 ES.reportError(Err: NewR.takeError());
413 R->failMaterialization();
414 return;
415 }
416
417 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
418 args: std::move(QueryAliases));
419 QueryInfos.push_back(
420 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
421 }
422
423 // Issue the queries.
424 while (!QueryInfos.empty()) {
425 auto QuerySymbols = std::move(QueryInfos.back().first);
426 auto QueryInfo = std::move(QueryInfos.back().second);
427
428 QueryInfos.pop_back();
429
430 auto RegisterDependencies = [QueryInfo,
431 &SrcJD](const SymbolDependenceMap &Deps) {
432 // If there were no materializing symbols, just bail out.
433 if (Deps.empty())
434 return;
435
436 // Otherwise the only deps should be on SrcJD.
437 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
438 "Unexpected dependencies for reexports");
439
440 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
441
442 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
443 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
444 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
445 };
446
447 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
448 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
449 if (Result) {
450 SymbolMap ResolutionMap;
451 for (auto &KV : QueryInfo->Aliases) {
452 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
453 Result->count(KV.second.Aliasee)) &&
454 "Result map missing entry?");
455 // Don't try to resolve materialization-side-effects-only symbols.
456 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
457 continue;
458
459 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
460 KV.second.AliasFlags};
461 }
462 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
463 ES.reportError(Err: std::move(Err));
464 QueryInfo->R->failMaterialization();
465 return;
466 }
467 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
468 ES.reportError(Err: std::move(Err));
469 QueryInfo->R->failMaterialization();
470 return;
471 }
472 } else {
473 ES.reportError(Err: Result.takeError());
474 QueryInfo->R->failMaterialization();
475 }
476 };
477
478 ES.lookup(K: LookupKind::Static,
479 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
480 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
481 RegisterDependencies: std::move(RegisterDependencies));
482 }
483}
484
485void ReExportsMaterializationUnit::discard(const JITDylib &JD,
486 const SymbolStringPtr &Name) {
487 assert(Aliases.count(Name) &&
488 "Symbol not covered by this MaterializationUnit");
489 Aliases.erase(Val: Name);
490}
491
492MaterializationUnit::Interface
493ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
494 SymbolFlagsMap SymbolFlags;
495 for (auto &KV : Aliases)
496 SymbolFlags[KV.first] = KV.second.AliasFlags;
497
498 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
499}
500
501Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
502 SymbolNameSet Symbols) {
503 SymbolLookupSet LookupSet(Symbols);
504 auto Flags = SourceJD.getExecutionSession().lookupFlags(
505 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
506 Symbols: SymbolLookupSet(std::move(Symbols)));
507
508 if (!Flags)
509 return Flags.takeError();
510
511 SymbolAliasMap Result;
512 for (auto &Name : Symbols) {
513 assert(Flags->count(Name) && "Missing entry in flags map");
514 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
515 }
516
517 return Result;
518}
519
520class InProgressLookupState {
521public:
522 // FIXME: Reduce the number of SymbolStringPtrs here. See
523 // https://github.com/llvm/llvm-project/issues/55576.
524
525 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
526 SymbolLookupSet LookupSet, SymbolState RequiredState)
527 : K(K), SearchOrder(std::move(SearchOrder)),
528 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
529 DefGeneratorCandidates = this->LookupSet;
530 }
531 virtual ~InProgressLookupState() = default;
532 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
533 virtual void fail(Error Err) = 0;
534
535 LookupKind K;
536 JITDylibSearchOrder SearchOrder;
537 SymbolLookupSet LookupSet;
538 SymbolState RequiredState;
539
540 size_t CurSearchOrderIndex = 0;
541 bool NewJITDylib = true;
542 SymbolLookupSet DefGeneratorCandidates;
543 SymbolLookupSet DefGeneratorNonCandidates;
544
545 enum {
546 NotInGenerator, // Not currently using a generator.
547 ResumedForGenerator, // Resumed after being auto-suspended before generator.
548 InGenerator // Currently using generator.
549 } GenState = NotInGenerator;
550 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
551};
552
553class InProgressLookupFlagsState : public InProgressLookupState {
554public:
555 InProgressLookupFlagsState(
556 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
557 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
558 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
559 SymbolState::NeverSearched),
560 OnComplete(std::move(OnComplete)) {}
561
562 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
563 auto &ES = SearchOrder.front().first->getExecutionSession();
564 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
565 }
566
567 void fail(Error Err) override { OnComplete(std::move(Err)); }
568
569private:
570 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
571};
572
573class InProgressFullLookupState : public InProgressLookupState {
574public:
575 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
576 SymbolLookupSet LookupSet,
577 SymbolState RequiredState,
578 std::shared_ptr<AsynchronousSymbolQuery> Q,
579 RegisterDependenciesFunction RegisterDependencies)
580 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
581 RequiredState),
582 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
583 }
584
585 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
586 auto &ES = SearchOrder.front().first->getExecutionSession();
587 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
588 RegisterDependencies: std::move(RegisterDependencies));
589 }
590
591 void fail(Error Err) override {
592 Q->detach();
593 Q->handleFailed(Err: std::move(Err));
594 }
595
596private:
597 std::shared_ptr<AsynchronousSymbolQuery> Q;
598 RegisterDependenciesFunction RegisterDependencies;
599};
600
601ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
602 JITDylibLookupFlags SourceJDLookupFlags,
603 SymbolPredicate Allow)
604 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
605 Allow(std::move(Allow)) {}
606
607Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
608 JITDylib &JD,
609 JITDylibLookupFlags JDLookupFlags,
610 const SymbolLookupSet &LookupSet) {
611 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
612
613 // Use lookupFlags to find the subset of symbols that match our lookup.
614 auto Flags = JD.getExecutionSession().lookupFlags(
615 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
616 if (!Flags)
617 return Flags.takeError();
618
619 // Create an alias map.
620 orc::SymbolAliasMap AliasMap;
621 for (auto &KV : *Flags)
622 if (!Allow || Allow(KV.first))
623 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
624
625 if (AliasMap.empty())
626 return Error::success();
627
628 // Define the re-exports.
629 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
630}
631
632LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
633 : IPLS(std::move(IPLS)) {}
634
635void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
636
637LookupState::LookupState() = default;
638LookupState::LookupState(LookupState &&) = default;
639LookupState &LookupState::operator=(LookupState &&) = default;
640LookupState::~LookupState() = default;
641
642void LookupState::continueLookup(Error Err) {
643 assert(IPLS && "Cannot call continueLookup on empty LookupState");
644 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
645 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
646}
647
648DefinitionGenerator::~DefinitionGenerator() {
649 std::deque<LookupState> LookupsToFail;
650 {
651 std::lock_guard<std::mutex> Lock(M);
652 std::swap(x&: PendingLookups, y&: LookupsToFail);
653 InUse = false;
654 }
655
656 for (auto &LS : LookupsToFail)
657 LS.continueLookup(Err: make_error<StringError>(
658 Args: "Query waiting on DefinitionGenerator that was destroyed",
659 Args: inconvertibleErrorCode()));
660}
661
662JITDylib::~JITDylib() {
663 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
664}
665
666Error JITDylib::clear() {
667 std::vector<ResourceTrackerSP> TrackersToRemove;
668 ES.runSessionLocked(F: [&]() {
669 assert(State != Closed && "JD is defunct");
670 for (auto &KV : TrackerSymbols)
671 TrackersToRemove.push_back(x: KV.first);
672 TrackersToRemove.push_back(x: getDefaultResourceTracker());
673 });
674
675 Error Err = Error::success();
676 for (auto &RT : TrackersToRemove)
677 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
678 return Err;
679}
680
681ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
682 return ES.runSessionLocked(F: [this] {
683 assert(State != Closed && "JD is defunct");
684 if (!DefaultTracker)
685 DefaultTracker = new ResourceTracker(this);
686 return DefaultTracker;
687 });
688}
689
690ResourceTrackerSP JITDylib::createResourceTracker() {
691 return ES.runSessionLocked(F: [this] {
692 assert(State == Open && "JD is defunct");
693 ResourceTrackerSP RT = new ResourceTracker(this);
694 return RT;
695 });
696}
697
698void JITDylib::removeGenerator(DefinitionGenerator &G) {
699 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
700 // session lock (since it may have to send errors to pending queries).
701 std::shared_ptr<DefinitionGenerator> TmpDG;
702
703 ES.runSessionLocked(F: [&] {
704 assert(State == Open && "JD is defunct");
705 auto I = llvm::find_if(Range&: DefGenerators,
706 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
707 return H.get() == &G;
708 });
709 assert(I != DefGenerators.end() && "Generator not found");
710 TmpDG = std::move(*I);
711 DefGenerators.erase(position: I);
712 });
713}
714
715Expected<SymbolFlagsMap>
716JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
717 SymbolFlagsMap SymbolFlags) {
718
719 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
720 if (FromMR.RT->isDefunct())
721 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
722
723 std::vector<NonOwningSymbolStringPtr> AddedSyms;
724 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
725
726 for (auto &[Name, Flags] : SymbolFlags) {
727 auto EntryItr = Symbols.find(Val: Name);
728
729 // If the entry already exists...
730 if (EntryItr != Symbols.end()) {
731
732 // If this is a strong definition then error out.
733 if (!Flags.isWeak()) {
734 // Remove any symbols already added.
735 for (auto &S : AddedSyms)
736 Symbols.erase(I: Symbols.find_as(Val: S));
737
738 // FIXME: Return all duplicates.
739 return make_error<DuplicateDefinition>(
740 Args: std::string(*Name), Args: "defineMaterializing operation");
741 }
742
743 // Otherwise just make a note to discard this symbol after the loop.
744 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
745 continue;
746 } else
747 EntryItr =
748 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
749
750 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
751 EntryItr->second.setState(SymbolState::Materializing);
752 }
753
754 // Remove any rejected weak definitions from the SymbolFlags map.
755 while (!RejectedWeakDefs.empty()) {
756 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
757 RejectedWeakDefs.pop_back();
758 }
759
760 return SymbolFlags;
761 });
762}
763
764Error JITDylib::replace(MaterializationResponsibility &FromMR,
765 std::unique_ptr<MaterializationUnit> MU) {
766 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
767 std::unique_ptr<MaterializationUnit> MustRunMU;
768 std::unique_ptr<MaterializationResponsibility> MustRunMR;
769
770 auto Err =
771 ES.runSessionLocked(F: [&, this]() -> Error {
772 if (FromMR.RT->isDefunct())
773 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
774
775#ifndef NDEBUG
776 for (auto &KV : MU->getSymbols()) {
777 auto SymI = Symbols.find(KV.first);
778 assert(SymI != Symbols.end() && "Replacing unknown symbol");
779 assert(SymI->second.getState() == SymbolState::Materializing &&
780 "Can not replace a symbol that ha is not materializing");
781 assert(!SymI->second.hasMaterializerAttached() &&
782 "Symbol should not have materializer attached already");
783 assert(UnmaterializedInfos.count(KV.first) == 0 &&
784 "Symbol being replaced should have no UnmaterializedInfo");
785 }
786#endif // NDEBUG
787
788 // If the tracker is defunct we need to bail out immediately.
789
790 // If any symbol has pending queries against it then we need to
791 // materialize MU immediately.
792 for (auto &KV : MU->getSymbols()) {
793 auto MII = MaterializingInfos.find(Val: KV.first);
794 if (MII != MaterializingInfos.end()) {
795 if (MII->second.hasQueriesPending()) {
796 MustRunMR = ES.createMaterializationResponsibility(
797 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
798 InitSymbol: std::move(MU->InitSymbol));
799 MustRunMU = std::move(MU);
800 return Error::success();
801 }
802 }
803 }
804
805 // Otherwise, make MU responsible for all the symbols.
806 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
807 args: FromMR.RT.get());
808 for (auto &KV : UMI->MU->getSymbols()) {
809 auto SymI = Symbols.find(Val: KV.first);
810 assert(SymI->second.getState() == SymbolState::Materializing &&
811 "Can not replace a symbol that is not materializing");
812 assert(!SymI->second.hasMaterializerAttached() &&
813 "Can not replace a symbol that has a materializer attached");
814 assert(UnmaterializedInfos.count(KV.first) == 0 &&
815 "Unexpected materializer entry in map");
816 SymI->second.setAddress(SymI->second.getAddress());
817 SymI->second.setMaterializerAttached(true);
818
819 auto &UMIEntry = UnmaterializedInfos[KV.first];
820 assert((!UMIEntry || !UMIEntry->MU) &&
821 "Replacing symbol with materializer still attached");
822 UMIEntry = UMI;
823 }
824
825 return Error::success();
826 });
827
828 if (Err)
829 return Err;
830
831 if (MustRunMU) {
832 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
833 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
834 args: std::move(MustRunMU), args: std::move(MustRunMR)));
835 } else {
836 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
837 }
838
839 return Error::success();
840}
841
842Expected<std::unique_ptr<MaterializationResponsibility>>
843JITDylib::delegate(MaterializationResponsibility &FromMR,
844 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
845
846 return ES.runSessionLocked(
847 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
848 if (FromMR.RT->isDefunct())
849 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
850
851 return ES.createMaterializationResponsibility(
852 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
853 });
854}
855
856SymbolNameSet
857JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
858 return ES.runSessionLocked(F: [&]() {
859 SymbolNameSet RequestedSymbols;
860
861 for (auto &KV : SymbolFlags) {
862 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
863 assert(Symbols.find(KV.first)->second.getState() !=
864 SymbolState::NeverSearched &&
865 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
866 "getRequestedSymbols can only be called for symbols that have "
867 "started materializing");
868 auto I = MaterializingInfos.find(Val: KV.first);
869 if (I == MaterializingInfos.end())
870 continue;
871
872 if (I->second.hasQueriesPending())
873 RequestedSymbols.insert(V: KV.first);
874 }
875
876 return RequestedSymbols;
877 });
878}
879
880Error JITDylib::resolve(MaterializationResponsibility &MR,
881 const SymbolMap &Resolved) {
882 AsynchronousSymbolQuerySet CompletedQueries;
883
884 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
885 if (MR.RT->isDefunct())
886 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
887
888 if (State != Open)
889 return make_error<StringError>(Args: "JITDylib " + getName() +
890 " is defunct",
891 Args: inconvertibleErrorCode());
892
893 struct WorklistEntry {
894 SymbolTable::iterator SymI;
895 ExecutorSymbolDef ResolvedSym;
896 };
897
898 SymbolNameSet SymbolsInErrorState;
899 std::vector<WorklistEntry> Worklist;
900 Worklist.reserve(n: Resolved.size());
901
902 // Build worklist and check for any symbols in the error state.
903 for (const auto &KV : Resolved) {
904
905 assert(!KV.second.getFlags().hasError() &&
906 "Resolution result can not have error flag set");
907
908 auto SymI = Symbols.find(Val: KV.first);
909
910 assert(SymI != Symbols.end() && "Symbol not found");
911 assert(!SymI->second.hasMaterializerAttached() &&
912 "Resolving symbol with materializer attached?");
913 assert(SymI->second.getState() == SymbolState::Materializing &&
914 "Symbol should be materializing");
915 assert(SymI->second.getAddress() == ExecutorAddr() &&
916 "Symbol has already been resolved");
917
918 if (SymI->second.getFlags().hasError())
919 SymbolsInErrorState.insert(V: KV.first);
920 else {
921 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
922 [[maybe_unused]] auto WeakOrCommon =
923 JITSymbolFlags::Weak | JITSymbolFlags::Common;
924 assert((KV.second.getFlags() & WeakOrCommon) &&
925 "Common symbols must be resolved as common or weak");
926 assert((KV.second.getFlags() & ~WeakOrCommon) ==
927 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
928 "Resolving symbol with incorrect flags");
929
930 } else
931 assert(KV.second.getFlags() == SymI->second.getFlags() &&
932 "Resolved flags should match the declared flags");
933
934 Worklist.push_back(
935 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
936 }
937 }
938
939 // If any symbols were in the error state then bail out.
940 if (!SymbolsInErrorState.empty()) {
941 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
942 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
943 return make_error<FailedToMaterialize>(
944 Args: getExecutionSession().getSymbolStringPool(),
945 Args: std::move(FailedSymbolsDepMap));
946 }
947
948 while (!Worklist.empty()) {
949 auto SymI = Worklist.back().SymI;
950 auto ResolvedSym = Worklist.back().ResolvedSym;
951 Worklist.pop_back();
952
953 auto &Name = SymI->first;
954
955 // Resolved symbols can not be weak: discard the weak flag.
956 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
957 SymI->second.setAddress(ResolvedSym.getAddress());
958 SymI->second.setFlags(ResolvedFlags);
959 SymI->second.setState(SymbolState::Resolved);
960
961 auto MII = MaterializingInfos.find(Val: Name);
962 if (MII == MaterializingInfos.end())
963 continue;
964
965 auto &MI = MII->second;
966 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
967 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
968 if (Q->isComplete())
969 CompletedQueries.insert(x: std::move(Q));
970 }
971 }
972
973 return Error::success();
974 }))
975 return Err;
976
977 // Otherwise notify all the completed queries.
978 for (auto &Q : CompletedQueries) {
979 assert(Q->isComplete() && "Q not completed");
980 Q->handleComplete(ES);
981 }
982
983 return Error::success();
984}
985
986void JITDylib::unlinkMaterializationResponsibility(
987 MaterializationResponsibility &MR) {
988 ES.runSessionLocked(F: [&]() {
989 auto I = TrackerMRs.find(Val: MR.RT.get());
990 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
991 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
992 I->second.erase(V: &MR);
993 if (I->second.empty())
994 TrackerMRs.erase(Val: MR.RT.get());
995 });
996}
997
998void JITDylib::shrinkMaterializationInfoMemory() {
999 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1000 // memory since we may have long-lived JDs after linking is done.
1001
1002 if (UnmaterializedInfos.empty())
1003 UnmaterializedInfos.clear();
1004
1005 if (MaterializingInfos.empty())
1006 MaterializingInfos.clear();
1007}
1008
1009void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1010 bool LinkAgainstThisJITDylibFirst) {
1011 ES.runSessionLocked(F: [&]() {
1012 assert(State == Open && "JD is defunct");
1013 if (LinkAgainstThisJITDylibFirst) {
1014 LinkOrder.clear();
1015 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1016 LinkOrder.push_back(
1017 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1018 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1019 } else
1020 LinkOrder = std::move(NewLinkOrder);
1021 });
1022}
1023
1024void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1025 ES.runSessionLocked(F: [&]() {
1026 for (auto &KV : NewLinks) {
1027 // Skip elements of NewLinks that are already in the link order.
1028 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1029 continue;
1030
1031 LinkOrder.push_back(x: std::move(KV));
1032 }
1033 });
1034}
1035
1036void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1037 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1038}
1039
1040void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1041 JITDylibLookupFlags JDLookupFlags) {
1042 ES.runSessionLocked(F: [&]() {
1043 assert(State == Open && "JD is defunct");
1044 for (auto &KV : LinkOrder)
1045 if (KV.first == &OldJD) {
1046 KV = {&NewJD, JDLookupFlags};
1047 break;
1048 }
1049 });
1050}
1051
1052void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1053 ES.runSessionLocked(F: [&]() {
1054 assert(State == Open && "JD is defunct");
1055 auto I = llvm::find_if(Range&: LinkOrder,
1056 P: [&](const JITDylibSearchOrder::value_type &KV) {
1057 return KV.first == &JD;
1058 });
1059 if (I != LinkOrder.end())
1060 LinkOrder.erase(position: I);
1061 });
1062}
1063
1064Error JITDylib::remove(const SymbolNameSet &Names) {
1065 return ES.runSessionLocked(F: [&]() -> Error {
1066 assert(State == Open && "JD is defunct");
1067 using SymbolMaterializerItrPair =
1068 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1069 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1070 SymbolNameSet Missing;
1071 SymbolNameSet Materializing;
1072
1073 for (auto &Name : Names) {
1074 auto I = Symbols.find(Val: Name);
1075
1076 // Note symbol missing.
1077 if (I == Symbols.end()) {
1078 Missing.insert(V: Name);
1079 continue;
1080 }
1081
1082 // Note symbol materializing.
1083 if (I->second.getState() != SymbolState::NeverSearched &&
1084 I->second.getState() != SymbolState::Ready) {
1085 Materializing.insert(V: Name);
1086 continue;
1087 }
1088
1089 auto UMII = I->second.hasMaterializerAttached()
1090 ? UnmaterializedInfos.find(Val: Name)
1091 : UnmaterializedInfos.end();
1092 SymbolsToRemove.push_back(x: std::make_pair(x&: I, y&: UMII));
1093 }
1094
1095 // If any of the symbols are not defined, return an error.
1096 if (!Missing.empty())
1097 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1098 Args: std::move(Missing));
1099
1100 // If any of the symbols are currently materializing, return an error.
1101 if (!Materializing.empty())
1102 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1103 Args: std::move(Materializing));
1104
1105 // Remove the symbols.
1106 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1107 auto UMII = SymbolMaterializerItrPair.second;
1108
1109 // If there is a materializer attached, call discard.
1110 if (UMII != UnmaterializedInfos.end()) {
1111 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1112 UnmaterializedInfos.erase(I: UMII);
1113 }
1114
1115 auto SymI = SymbolMaterializerItrPair.first;
1116 Symbols.erase(I: SymI);
1117 }
1118
1119 shrinkMaterializationInfoMemory();
1120
1121 return Error::success();
1122 });
1123}
1124
1125void JITDylib::dump(raw_ostream &OS) {
1126 ES.runSessionLocked(F: [&, this]() {
1127 OS << "JITDylib \"" << getName() << "\" (ES: "
1128 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1129 << ", State = ";
1130 switch (State) {
1131 case Open:
1132 OS << "Open";
1133 break;
1134 case Closing:
1135 OS << "Closing";
1136 break;
1137 case Closed:
1138 OS << "Closed";
1139 break;
1140 }
1141 OS << ")\n";
1142 if (State == Closed)
1143 return;
1144 OS << "Link order: " << LinkOrder << "\n"
1145 << "Symbol table:\n";
1146
1147 // Sort symbols so we get a deterministic order and can check them in tests.
1148 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1149 for (auto &KV : Symbols)
1150 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1151 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1152 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1153
1154 for (auto &KV : SymbolsSorted) {
1155 OS << " \"" << *KV.first << "\": ";
1156 if (auto Addr = KV.second->getAddress())
1157 OS << Addr;
1158 else
1159 OS << "<not resolved> ";
1160
1161 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1162
1163 if (KV.second->hasMaterializerAttached()) {
1164 OS << " (Materializer ";
1165 auto I = UnmaterializedInfos.find(Val: KV.first);
1166 assert(I != UnmaterializedInfos.end() &&
1167 "Lazy symbol should have UnmaterializedInfo");
1168 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1169 } else
1170 OS << "\n";
1171 }
1172
1173 if (!MaterializingInfos.empty())
1174 OS << " MaterializingInfos entries:\n";
1175 for (auto &KV : MaterializingInfos) {
1176 OS << " \"" << *KV.first << "\":\n"
1177 << " " << KV.second.pendingQueries().size()
1178 << " pending queries: { ";
1179 for (const auto &Q : KV.second.pendingQueries())
1180 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1181 OS << "}\n";
1182 }
1183 });
1184}
1185
1186void JITDylib::MaterializingInfo::addQuery(
1187 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1188
1189 auto I = llvm::lower_bound(
1190 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1191 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1192 return V->getRequiredState() <= S;
1193 });
1194 PendingQueries.insert(position: I.base(), x: std::move(Q));
1195}
1196
1197void JITDylib::MaterializingInfo::removeQuery(
1198 const AsynchronousSymbolQuery &Q) {
1199 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1200 auto I = llvm::find_if(
1201 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1202 return V.get() == &Q;
1203 });
1204 if (I != PendingQueries.end())
1205 PendingQueries.erase(position: I);
1206}
1207
1208JITDylib::AsynchronousSymbolQueryList
1209JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1210 AsynchronousSymbolQueryList Result;
1211 while (!PendingQueries.empty()) {
1212 if (PendingQueries.back()->getRequiredState() > RequiredState)
1213 break;
1214
1215 Result.push_back(x: std::move(PendingQueries.back()));
1216 PendingQueries.pop_back();
1217 }
1218
1219 return Result;
1220}
1221
1222JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1223 : JITLinkDylib(std::move(Name)), ES(ES) {
1224 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1225}
1226
1227JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1228 // Note: Should be called under the session lock.
1229 assert(State != Closed && "JD is defunct");
1230
1231 SymbolNameVector SymbolsToRemove;
1232 SymbolNameVector SymbolsToFail;
1233
1234 if (&RT == DefaultTracker.get()) {
1235 SymbolNameSet TrackedSymbols;
1236 for (auto &KV : TrackerSymbols)
1237 TrackedSymbols.insert_range(R&: KV.second);
1238
1239 for (auto &KV : Symbols) {
1240 auto &Sym = KV.first;
1241 if (!TrackedSymbols.count(V: Sym))
1242 SymbolsToRemove.push_back(x: Sym);
1243 }
1244
1245 DefaultTracker.reset();
1246 } else {
1247 /// Check for a non-default tracker.
1248 auto I = TrackerSymbols.find(Val: &RT);
1249 if (I != TrackerSymbols.end()) {
1250 SymbolsToRemove = std::move(I->second);
1251 TrackerSymbols.erase(I);
1252 }
1253 // ... if not found this tracker was already defunct. Nothing to do.
1254 }
1255
1256 for (auto &Sym : SymbolsToRemove) {
1257 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1258
1259 // If there is a MaterializingInfo then collect any queries to fail.
1260 auto MII = MaterializingInfos.find(Val: Sym);
1261 if (MII != MaterializingInfos.end())
1262 SymbolsToFail.push_back(x: Sym);
1263 }
1264
1265 auto [QueriesToFail, FailedSymbols] =
1266 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1267
1268 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1269
1270 // Removed symbols should be taken out of the table altogether.
1271 for (auto &Sym : SymbolsToRemove) {
1272 auto I = Symbols.find(Val: Sym);
1273 assert(I != Symbols.end() && "Symbol not present in table");
1274
1275 // Remove Materializer if present.
1276 if (I->second.hasMaterializerAttached()) {
1277 // FIXME: Should this discard the symbols?
1278 auto J = UnmaterializedInfos.find(Val: Sym);
1279 assert(J != UnmaterializedInfos.end() &&
1280 "Symbol table indicates MU present, but no UMI record");
1281 if (J->second->MU)
1282 DefunctMUs.push_back(x: std::move(J->second->MU));
1283 UnmaterializedInfos.erase(I: J);
1284 } else {
1285 assert(!UnmaterializedInfos.count(Sym) &&
1286 "Symbol has materializer attached");
1287 }
1288
1289 Symbols.erase(I);
1290 }
1291
1292 shrinkMaterializationInfoMemory();
1293
1294 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1295 .DefunctMUs: std::move(DefunctMUs)};
1296}
1297
1298void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1299 assert(State != Closed && "JD is defunct");
1300 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1301 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1302 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1303
1304 // Update trackers for any not-yet materialized units.
1305 for (auto &KV : UnmaterializedInfos) {
1306 if (KV.second->RT == &SrcRT)
1307 KV.second->RT = &DstRT;
1308 }
1309
1310 // Update trackers for any active materialization responsibilities.
1311 {
1312 auto I = TrackerMRs.find(Val: &SrcRT);
1313 if (I != TrackerMRs.end()) {
1314 auto &SrcMRs = I->second;
1315 auto &DstMRs = TrackerMRs[&DstRT];
1316 for (auto *MR : SrcMRs)
1317 MR->RT = &DstRT;
1318 if (DstMRs.empty())
1319 DstMRs = std::move(SrcMRs);
1320 else
1321 DstMRs.insert_range(R&: SrcMRs);
1322 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1323 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1324 TrackerMRs.erase(Val: &SrcRT);
1325 }
1326 }
1327
1328 // If we're transfering to the default tracker we just need to delete the
1329 // tracked symbols for the source tracker.
1330 if (&DstRT == DefaultTracker.get()) {
1331 TrackerSymbols.erase(Val: &SrcRT);
1332 return;
1333 }
1334
1335 // If we're transferring from the default tracker we need to find all
1336 // currently untracked symbols.
1337 if (&SrcRT == DefaultTracker.get()) {
1338 assert(!TrackerSymbols.count(&SrcRT) &&
1339 "Default tracker should not appear in TrackerSymbols");
1340
1341 SymbolNameVector SymbolsToTrack;
1342
1343 SymbolNameSet CurrentlyTrackedSymbols;
1344 for (auto &KV : TrackerSymbols)
1345 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1346
1347 for (auto &KV : Symbols) {
1348 auto &Sym = KV.first;
1349 if (!CurrentlyTrackedSymbols.count(V: Sym))
1350 SymbolsToTrack.push_back(x: Sym);
1351 }
1352
1353 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1354 return;
1355 }
1356
1357 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1358
1359 // Finally if neither SrtRT or DstRT are the default tracker then
1360 // just append DstRT's tracked symbols to SrtRT's.
1361 auto SI = TrackerSymbols.find(Val: &SrcRT);
1362 if (SI == TrackerSymbols.end())
1363 return;
1364
1365 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1366 for (auto &Sym : SI->second)
1367 DstTrackedSymbols.push_back(x: std::move(Sym));
1368 TrackerSymbols.erase(I: SI);
1369}
1370
1371Error JITDylib::defineImpl(MaterializationUnit &MU) {
1372 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1373
1374 SymbolNameSet Duplicates;
1375 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1376 std::vector<SymbolStringPtr> MUDefsOverridden;
1377
1378 for (const auto &KV : MU.getSymbols()) {
1379 auto I = Symbols.find(Val: KV.first);
1380
1381 if (I != Symbols.end()) {
1382 if (KV.second.isStrong()) {
1383 if (I->second.getFlags().isStrong() ||
1384 I->second.getState() > SymbolState::NeverSearched)
1385 Duplicates.insert(V: KV.first);
1386 else {
1387 assert(I->second.getState() == SymbolState::NeverSearched &&
1388 "Overridden existing def should be in the never-searched "
1389 "state");
1390 ExistingDefsOverridden.push_back(x: KV.first);
1391 }
1392 } else
1393 MUDefsOverridden.push_back(x: KV.first);
1394 }
1395 }
1396
1397 // If there were any duplicate definitions then bail out.
1398 if (!Duplicates.empty()) {
1399 LLVM_DEBUG(
1400 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1401 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1402 Args: MU.getName().str());
1403 }
1404
1405 // Discard any overridden defs in this MU.
1406 LLVM_DEBUG({
1407 if (!MUDefsOverridden.empty())
1408 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1409 });
1410 for (auto &S : MUDefsOverridden)
1411 MU.doDiscard(JD: *this, Name: S);
1412
1413 // Discard existing overridden defs.
1414 LLVM_DEBUG({
1415 if (!ExistingDefsOverridden.empty())
1416 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1417 << "\n";
1418 });
1419 for (auto &S : ExistingDefsOverridden) {
1420
1421 auto UMII = UnmaterializedInfos.find(Val: S);
1422 assert(UMII != UnmaterializedInfos.end() &&
1423 "Overridden existing def should have an UnmaterializedInfo");
1424 UMII->second->MU->doDiscard(JD: *this, Name: S);
1425 }
1426
1427 // Finally, add the defs from this MU.
1428 for (auto &KV : MU.getSymbols()) {
1429 auto &SymEntry = Symbols[KV.first];
1430 SymEntry.setFlags(KV.second);
1431 SymEntry.setState(SymbolState::NeverSearched);
1432 SymEntry.setMaterializerAttached(true);
1433 }
1434
1435 return Error::success();
1436}
1437
1438void JITDylib::installMaterializationUnit(
1439 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1440
1441 /// defineImpl succeeded.
1442 if (&RT != DefaultTracker.get()) {
1443 auto &TS = TrackerSymbols[&RT];
1444 TS.reserve(n: TS.size() + MU->getSymbols().size());
1445 for (auto &KV : MU->getSymbols())
1446 TS.push_back(x: KV.first);
1447 }
1448
1449 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1450 for (auto &KV : UMI->MU->getSymbols())
1451 UnmaterializedInfos[KV.first] = UMI;
1452}
1453
1454void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1455 const SymbolNameSet &QuerySymbols) {
1456 for (auto &QuerySymbol : QuerySymbols) {
1457 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1458 if (MII != MaterializingInfos.end())
1459 MII->second.removeQuery(Q);
1460 }
1461}
1462
1463Platform::~Platform() = default;
1464
1465Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1466 ExecutionSession &ES,
1467 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1468
1469 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1470 Error CompoundErr = Error::success();
1471 std::mutex LookupMutex;
1472 std::condition_variable CV;
1473 uint64_t Count = InitSyms.size();
1474
1475 LLVM_DEBUG({
1476 dbgs() << "Issuing init-symbol lookup:\n";
1477 for (auto &KV : InitSyms)
1478 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1479 });
1480
1481 for (auto &KV : InitSyms) {
1482 auto *JD = KV.first;
1483 auto Names = std::move(KV.second);
1484 ES.lookup(
1485 K: LookupKind::Static,
1486 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1487 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1488 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1489 {
1490 std::lock_guard<std::mutex> Lock(LookupMutex);
1491 --Count;
1492 if (Result) {
1493 assert(!CompoundResult.count(JD) &&
1494 "Duplicate JITDylib in lookup?");
1495 CompoundResult[JD] = std::move(*Result);
1496 } else
1497 CompoundErr =
1498 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1499 }
1500 CV.notify_one();
1501 },
1502 RegisterDependencies: NoDependenciesToRegister);
1503 }
1504
1505 std::unique_lock<std::mutex> Lock(LookupMutex);
1506 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1507
1508 if (CompoundErr)
1509 return std::move(CompoundErr);
1510
1511 return std::move(CompoundResult);
1512}
1513
1514void Platform::lookupInitSymbolsAsync(
1515 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1516 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1517
1518 class TriggerOnComplete {
1519 public:
1520 using OnCompleteFn = unique_function<void(Error)>;
1521 TriggerOnComplete(OnCompleteFn OnComplete)
1522 : OnComplete(std::move(OnComplete)) {}
1523 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1524 void reportResult(Error Err) {
1525 std::lock_guard<std::mutex> Lock(ResultMutex);
1526 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1527 }
1528
1529 private:
1530 std::mutex ResultMutex;
1531 Error LookupResult{Error::success()};
1532 OnCompleteFn OnComplete;
1533 };
1534
1535 LLVM_DEBUG({
1536 dbgs() << "Issuing init-symbol lookup:\n";
1537 for (auto &KV : InitSyms)
1538 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1539 });
1540
1541 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1542
1543 for (auto &KV : InitSyms) {
1544 auto *JD = KV.first;
1545 auto Names = std::move(KV.second);
1546 ES.lookup(
1547 K: LookupKind::Static,
1548 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1549 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1550 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1551 TOC->reportResult(Err: Result.takeError());
1552 },
1553 RegisterDependencies: NoDependenciesToRegister);
1554 }
1555}
1556
1557MaterializationTask::~MaterializationTask() {
1558 // If this task wasn't run then fail materialization.
1559 if (MR)
1560 MR->failMaterialization();
1561}
1562
1563void MaterializationTask::printDescription(raw_ostream &OS) {
1564 OS << "Materialization task: " << MU->getName() << " in "
1565 << MR->getTargetJITDylib().getName();
1566}
1567
1568void MaterializationTask::run() {
1569 assert(MU && "MU should not be null");
1570 assert(MR && "MR should not be null");
1571 MU->materialize(R: std::move(MR));
1572}
1573
1574void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1575
1576void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1577
1578ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1579 : EPC(std::move(EPC)) {
1580 // Associated EPC and this.
1581 this->EPC->ES = this;
1582}
1583
1584ExecutionSession::~ExecutionSession() {
1585 // You must call endSession prior to destroying the session.
1586 assert(!SessionOpen &&
1587 "Session still open. Did you forget to call endSession?");
1588}
1589
1590Error ExecutionSession::endSession() {
1591 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1592
1593 WaitingOnGraph::OpRecorder *GOpRecorderToEnd = nullptr;
1594 auto JDsToRemove = runSessionLocked(F: [&] {
1595
1596#ifdef EXPENSIVE_CHECKS
1597 verifySessionState("Entering ExecutionSession::endSession");
1598#endif
1599
1600 if (SessionOpen)
1601 GOpRecorderToEnd = GOpRecorder;
1602 SessionOpen = false;
1603 return JDs;
1604 });
1605
1606 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1607
1608 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1609
1610 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1611
1612 if (GOpRecorderToEnd)
1613 GOpRecorderToEnd->recordEnd();
1614
1615 return Err;
1616}
1617
1618void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1619 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1620}
1621
1622void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1623 runSessionLocked(F: [&] {
1624 assert(!ResourceManagers.empty() && "No managers registered");
1625 if (ResourceManagers.back() == &RM)
1626 ResourceManagers.pop_back();
1627 else {
1628 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1629 assert(I != ResourceManagers.end() && "RM not registered");
1630 ResourceManagers.erase(position: I);
1631 }
1632 });
1633}
1634
1635JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1636 return runSessionLocked(F: [&, this]() -> JITDylib * {
1637 for (auto &JD : JDs)
1638 if (JD->getName() == Name)
1639 return JD.get();
1640 return nullptr;
1641 });
1642}
1643
1644JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1645 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1646 return runSessionLocked(F: [&, this]() -> JITDylib & {
1647 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1648 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1649 return *JDs.back();
1650 });
1651}
1652
1653Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1654 auto &JD = createBareJITDylib(Name);
1655 if (P)
1656 if (auto Err = P->setupJITDylib(JD))
1657 return std::move(Err);
1658 return JD;
1659}
1660
1661Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1662 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1663 runSessionLocked(F: [&] {
1664 for (auto &JD : JDsToRemove) {
1665 assert(JD->State == JITDylib::Open && "JD already closed");
1666 JD->State = JITDylib::Closing;
1667 auto I = llvm::find(Range&: JDs, Val: JD);
1668 assert(I != JDs.end() && "JD does not appear in session JDs");
1669 JDs.erase(position: I);
1670 }
1671 });
1672
1673 // Clear JITDylibs and notify the platform.
1674 Error Err = Error::success();
1675 for (auto JD : JDsToRemove) {
1676 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1677 if (P)
1678 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1679 }
1680
1681 // Set JD to closed state. Clear remaining data structures.
1682 runSessionLocked(F: [&] {
1683 for (auto &JD : JDsToRemove) {
1684 assert(JD->State == JITDylib::Closing && "JD should be closing");
1685 JD->State = JITDylib::Closed;
1686 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1687 assert(JD->UnmaterializedInfos.empty() &&
1688 "JD.UnmaterializedInfos is not empty after clear");
1689 assert(JD->MaterializingInfos.empty() &&
1690 "JD.MaterializingInfos is not empty after clear");
1691 assert(JD->TrackerSymbols.empty() &&
1692 "TrackerSymbols is not empty after clear");
1693 JD->DefGenerators.clear();
1694 JD->LinkOrder.clear();
1695 }
1696 });
1697
1698 return Err;
1699}
1700
1701Expected<std::vector<JITDylibSP>>
1702JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1703 if (JDs.empty())
1704 return std::vector<JITDylibSP>();
1705
1706 auto &ES = JDs.front()->getExecutionSession();
1707 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1708 DenseSet<JITDylib *> Visited;
1709 std::vector<JITDylibSP> Result;
1710
1711 for (auto &JD : JDs) {
1712
1713 if (JD->State != Open)
1714 return make_error<StringError>(
1715 Args: "Error building link order: " + JD->getName() + " is defunct",
1716 Args: inconvertibleErrorCode());
1717 if (Visited.count(V: JD.get()))
1718 continue;
1719
1720 SmallVector<JITDylibSP, 64> WorkStack;
1721 WorkStack.push_back(Elt: JD);
1722 Visited.insert(V: JD.get());
1723
1724 while (!WorkStack.empty()) {
1725 Result.push_back(x: std::move(WorkStack.back()));
1726 WorkStack.pop_back();
1727
1728 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1729 auto &JD = *KV.first;
1730 if (!Visited.insert(V: &JD).second)
1731 continue;
1732 WorkStack.push_back(Elt: &JD);
1733 }
1734 }
1735 }
1736 return Result;
1737 });
1738}
1739
1740Expected<std::vector<JITDylibSP>>
1741JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1742 auto Result = getDFSLinkOrder(JDs);
1743 if (Result)
1744 std::reverse(first: Result->begin(), last: Result->end());
1745 return Result;
1746}
1747
1748Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1749 return getDFSLinkOrder(JDs: {this});
1750}
1751
1752Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1753 return getReverseDFSLinkOrder(JDs: {this});
1754}
1755
1756void ExecutionSession::lookupFlags(
1757 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1758 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1759
1760 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1761 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1762 args: std::move(OnComplete)),
1763 Err: Error::success());
1764}
1765
1766Expected<SymbolFlagsMap>
1767ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1768 SymbolLookupSet LookupSet) {
1769
1770 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1771 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1772 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1773 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1774 ResultP.set_value(std::move(Result));
1775 }),
1776 Err: Error::success());
1777
1778 auto ResultF = ResultP.get_future();
1779 return ResultF.get();
1780}
1781
1782void ExecutionSession::lookup(
1783 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1784 SymbolLookupSet Symbols, SymbolState RequiredState,
1785 SymbolsResolvedCallback NotifyComplete,
1786 RegisterDependenciesFunction RegisterDependencies) {
1787
1788 LLVM_DEBUG({
1789 runSessionLocked([&]() {
1790 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1791 << " (required state: " << RequiredState << ")\n";
1792 });
1793 });
1794
1795 // lookup can be re-entered recursively if running on a single thread. Run any
1796 // outstanding MUs in case this query depends on them, otherwise this lookup
1797 // will starve waiting for a result from an MU that is stuck in the queue.
1798 dispatchOutstandingMUs();
1799
1800 auto Unresolved = std::move(Symbols);
1801 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1802 args: std::move(NotifyComplete));
1803
1804 auto IPLS = std::make_unique<InProgressFullLookupState>(
1805 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1806 args: std::move(RegisterDependencies));
1807
1808 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1809}
1810
1811Expected<SymbolMap>
1812ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1813 SymbolLookupSet Symbols, LookupKind K,
1814 SymbolState RequiredState,
1815 RegisterDependenciesFunction RegisterDependencies) {
1816#if LLVM_ENABLE_THREADS
1817 // In the threaded case we use promises to return the results.
1818 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1819
1820 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1821 PromisedResult.set_value(std::move(R));
1822 };
1823
1824#else
1825 SymbolMap Result;
1826 Error ResolutionError = Error::success();
1827
1828 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1829 ErrorAsOutParameter _(ResolutionError);
1830 if (R)
1831 Result = std::move(*R);
1832 else
1833 ResolutionError = R.takeError();
1834 };
1835#endif
1836
1837 // Perform the asynchronous lookup.
1838 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1839 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1840
1841#if LLVM_ENABLE_THREADS
1842 return PromisedResult.get_future().get();
1843#else
1844 if (ResolutionError)
1845 return std::move(ResolutionError);
1846
1847 return Result;
1848#endif
1849}
1850
1851Expected<ExecutorSymbolDef>
1852ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1853 SymbolStringPtr Name, SymbolState RequiredState) {
1854 SymbolLookupSet Names({Name});
1855
1856 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1857 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1858 assert(ResultMap->size() == 1 && "Unexpected number of results");
1859 assert(ResultMap->count(Name) && "Missing result for symbol");
1860 return std::move(ResultMap->begin()->second);
1861 } else
1862 return ResultMap.takeError();
1863}
1864
1865Expected<ExecutorSymbolDef>
1866ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1867 SymbolState RequiredState) {
1868 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1869}
1870
1871Expected<ExecutorSymbolDef>
1872ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1873 SymbolState RequiredState) {
1874 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1875}
1876
1877Error ExecutionSession::registerJITDispatchHandlers(
1878 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1879
1880 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1881 Symbols: SymbolLookupSet::fromMapKeys(
1882 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1883 if (!TagSyms)
1884 return TagSyms.takeError();
1885
1886 // Associate tag addresses with implementations.
1887 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1888
1889 // Check that no tags are being overwritten.
1890 for (auto &[TagName, TagSym] : *TagSyms) {
1891 auto TagAddr = TagSym.getAddress();
1892 if (JITDispatchHandlers.count(Val: TagAddr))
1893 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1894 " (for " + *TagName +
1895 ") already registered",
1896 Args: inconvertibleErrorCode());
1897 }
1898
1899 // At this point we're guaranteed to succeed. Install the handlers.
1900 for (auto &[TagName, TagSym] : *TagSyms) {
1901 auto TagAddr = TagSym.getAddress();
1902 auto I = WFs.find(Val: TagName);
1903 assert(I != WFs.end() && I->second &&
1904 "JITDispatchHandler implementation missing");
1905 JITDispatchHandlers[TagAddr] =
1906 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1907 LLVM_DEBUG({
1908 dbgs() << "Associated function tag \"" << *TagName << "\" ("
1909 << formatv("{0:x}", TagAddr) << ") with handler\n";
1910 });
1911 }
1912
1913 return Error::success();
1914}
1915
1916void ExecutionSession::runJITDispatchHandler(
1917 SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr,
1918 shared::WrapperFunctionBuffer ArgBytes) {
1919
1920 std::shared_ptr<JITDispatchHandlerFunction> F;
1921 {
1922 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1923 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1924 if (I != JITDispatchHandlers.end())
1925 F = I->second;
1926 }
1927
1928 if (F)
1929 (*F)(std::move(SendResult), ArgBytes.data(), ArgBytes.size());
1930 else
1931 SendResult(shared::WrapperFunctionBuffer::createOutOfBandError(
1932 Msg: ("No function registered for tag " +
1933 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1934 .str()));
1935}
1936
1937void ExecutionSession::dump(raw_ostream &OS) {
1938 runSessionLocked(F: [this, &OS]() {
1939 for (auto &JD : JDs)
1940 JD->dump(OS);
1941 });
1942}
1943
1944#ifdef EXPENSIVE_CHECKS
1945bool ExecutionSession::verifySessionState(Twine Phase) {
1946 return runSessionLocked([&]() {
1947 bool AllOk = true;
1948
1949 for (auto &JD : JDs) {
1950
1951 auto LogFailure = [&]() -> raw_fd_ostream & {
1952 auto &Stream = errs();
1953 if (AllOk)
1954 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1955 << "\n";
1956 Stream << " In JITDylib " << JD->getName() << ", ";
1957 AllOk = false;
1958 return Stream;
1959 };
1960
1961 if (JD->State != JITDylib::Open) {
1962 LogFailure()
1963 << "state is not Open, but JD is in ExecutionSession list.";
1964 }
1965
1966 // Check symbol table.
1967 // 1. If the entry state isn't resolved then check that no address has
1968 // been set.
1969 // 2. Check that if the hasMaterializerAttached flag is set then there is
1970 // an UnmaterializedInfo entry, and vice-versa.
1971 for (auto &[Sym, Entry] : JD->Symbols) {
1972 // Check that unresolved symbols have null addresses.
1973 if (Entry.getState() < SymbolState::Resolved) {
1974 if (Entry.getAddress()) {
1975 LogFailure() << "symbol " << Sym << " has state "
1976 << Entry.getState()
1977 << " (not-yet-resolved) but non-null address "
1978 << Entry.getAddress() << ".\n";
1979 }
1980 }
1981
1982 // Check that the hasMaterializerAttached flag is correct.
1983 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
1984 if (Entry.hasMaterializerAttached()) {
1985 if (UMIItr == JD->UnmaterializedInfos.end()) {
1986 LogFailure() << "symbol " << Sym
1987 << " entry claims materializer attached, but "
1988 "UnmaterializedInfos has no corresponding entry.\n";
1989 }
1990 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
1991 LogFailure()
1992 << "symbol " << Sym
1993 << " entry claims no materializer attached, but "
1994 "UnmaterializedInfos has an unexpected entry for it.\n";
1995 }
1996 }
1997
1998 // Check that every UnmaterializedInfo entry has a corresponding entry
1999 // in the Symbols table.
2000 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2001 auto SymItr = JD->Symbols.find(Sym);
2002 if (SymItr == JD->Symbols.end()) {
2003 LogFailure()
2004 << "symbol " << Sym
2005 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2006 }
2007 }
2008
2009 // Check consistency of the MaterializingInfos table.
2010 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2011
2012 auto SymItr = JD->Symbols.find(Sym);
2013 if (SymItr == JD->Symbols.end()) {
2014 // If there's no Symbols entry for this MaterializingInfos entry then
2015 // report that.
2016 LogFailure()
2017 << "symbol " << Sym
2018 << " has MaterializingInfos entry, but no Symbols entry.\n";
2019 } else {
2020 // Otherwise check consistency between Symbols and MaterializingInfos.
2021
2022 // Ready symbols should not have MaterializingInfos.
2023 if (SymItr->second.getState() == SymbolState::Ready) {
2024 LogFailure()
2025 << "symbol " << Sym
2026 << " is in Ready state, should not have MaterializingInfo.\n";
2027 }
2028
2029 // Pending queries should be for subsequent states.
2030 auto CurState = static_cast<SymbolState>(
2031 static_cast<std::underlying_type_t<SymbolState>>(
2032 SymItr->second.getState()) + 1);
2033 for (auto &Q : MII.PendingQueries) {
2034 if (Q->getRequiredState() != CurState) {
2035 if (Q->getRequiredState() > CurState)
2036 CurState = Q->getRequiredState();
2037 else
2038 LogFailure() << "symbol " << Sym
2039 << " has stale or misordered queries.\n";
2040 }
2041 }
2042 }
2043 }
2044 }
2045
2046 return AllOk;
2047 });
2048}
2049#endif // EXPENSIVE_CHECKS
2050
2051void ExecutionSession::dispatchOutstandingMUs() {
2052 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2053 while (true) {
2054 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2055 std::unique_ptr<MaterializationResponsibility>>>
2056 JMU;
2057
2058 {
2059 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2060 if (!OutstandingMUs.empty()) {
2061 JMU.emplace(args: std::move(OutstandingMUs.back()));
2062 OutstandingMUs.pop_back();
2063 }
2064 }
2065
2066 if (!JMU)
2067 break;
2068
2069 assert(JMU->first && "No MU?");
2070 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2071 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2072 args: std::move(JMU->second)));
2073 }
2074 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2075}
2076
2077Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2078 LLVM_DEBUG({
2079 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2080 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2081 });
2082 std::vector<ResourceManager *> CurrentResourceManagers;
2083
2084 JITDylib::RemoveTrackerResult R;
2085
2086 runSessionLocked(F: [&] {
2087 CurrentResourceManagers = ResourceManagers;
2088 RT.makeDefunct();
2089 R = RT.getJITDylib().IL_removeTracker(RT);
2090 });
2091
2092 // Release any defunct MaterializationUnits.
2093 R.DefunctMUs.clear();
2094
2095 Error Err = Error::success();
2096
2097 auto &JD = RT.getJITDylib();
2098 for (auto *L : reverse(C&: CurrentResourceManagers))
2099 Err = joinErrors(E1: std::move(Err),
2100 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2101
2102 for (auto &Q : R.QueriesToFail)
2103 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2104 Args&: R.FailedSymbols));
2105
2106 return Err;
2107}
2108
2109void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2110 ResourceTracker &SrcRT) {
2111 LLVM_DEBUG({
2112 dbgs() << "In " << SrcRT.getJITDylib().getName()
2113 << " transfering resources from tracker "
2114 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2115 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2116 });
2117
2118 // No-op transfers are allowed and do not invalidate the source.
2119 if (&DstRT == &SrcRT)
2120 return;
2121
2122 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2123 "Can't transfer resources between JITDylibs");
2124 runSessionLocked(F: [&]() {
2125 SrcRT.makeDefunct();
2126 auto &JD = DstRT.getJITDylib();
2127 JD.transferTracker(DstRT, SrcRT);
2128 for (auto *L : reverse(C&: ResourceManagers))
2129 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2130 SrcK: SrcRT.getKeyUnsafe());
2131 });
2132}
2133
2134void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2135 runSessionLocked(F: [&]() {
2136 LLVM_DEBUG({
2137 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2138 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2139 });
2140 if (!RT.isDefunct())
2141 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2142 SrcRT&: RT);
2143 });
2144}
2145
2146Error ExecutionSession::IL_updateCandidatesFor(
2147 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2148 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2149 return Candidates.forEachWithRemoval(
2150 Body: [&](const SymbolStringPtr &Name,
2151 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2152 /// Search for the symbol. If not found then continue without
2153 /// removal.
2154 auto SymI = JD.Symbols.find(Val: Name);
2155 if (SymI == JD.Symbols.end())
2156 return false;
2157
2158 // If this is a non-exported symbol and we're matching exported
2159 // symbols only then remove this symbol from the candidates list.
2160 //
2161 // If we're tracking non-candidates then add this to the non-candidate
2162 // list.
2163 if (!SymI->second.getFlags().isExported() &&
2164 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2165 if (NonCandidates)
2166 NonCandidates->add(Name, Flags: SymLookupFlags);
2167 return true;
2168 }
2169
2170 // If we match against a materialization-side-effects only symbol
2171 // then make sure it is weakly-referenced. Otherwise bail out with
2172 // an error.
2173 // FIXME: Use a "materialization-side-effects-only symbols must be
2174 // weakly referenced" specific error here to reduce confusion.
2175 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2176 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2177 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2178 Args: SymbolNameVector({Name}));
2179
2180 // If we matched against this symbol but it is in the error state
2181 // then bail out and treat it as a failure to materialize.
2182 if (SymI->second.getFlags().hasError()) {
2183 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2184 (*FailedSymbolsMap)[&JD] = {Name};
2185 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2186 Args: std::move(FailedSymbolsMap));
2187 }
2188
2189 // Otherwise this is a match. Remove it from the candidate set.
2190 return true;
2191 });
2192}
2193
2194void ExecutionSession::OL_resumeLookupAfterGeneration(
2195 InProgressLookupState &IPLS) {
2196
2197 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2198 "Should not be called for not-in-generator lookups");
2199 IPLS.GenState = InProgressLookupState::NotInGenerator;
2200
2201 LookupState LS;
2202
2203 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2204 IPLS.CurDefGeneratorStack.pop_back();
2205 std::lock_guard<std::mutex> Lock(DG->M);
2206
2207 // If there are no pending lookups then mark the generator as free and
2208 // return.
2209 if (DG->PendingLookups.empty()) {
2210 DG->InUse = false;
2211 return;
2212 }
2213
2214 // Otherwise resume the next lookup.
2215 LS = std::move(DG->PendingLookups.front());
2216 DG->PendingLookups.pop_front();
2217 }
2218
2219 if (LS.IPLS) {
2220 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2221 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2222 }
2223}
2224
2225void ExecutionSession::OL_applyQueryPhase1(
2226 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2227
2228 LLVM_DEBUG({
2229 dbgs() << "Entering OL_applyQueryPhase1:\n"
2230 << " Lookup kind: " << IPLS->K << "\n"
2231 << " Search order: " << IPLS->SearchOrder
2232 << ", Current index = " << IPLS->CurSearchOrderIndex
2233 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2234 << " Lookup set: " << IPLS->LookupSet << "\n"
2235 << " Definition generator candidates: "
2236 << IPLS->DefGeneratorCandidates << "\n"
2237 << " Definition generator non-candidates: "
2238 << IPLS->DefGeneratorNonCandidates << "\n";
2239 });
2240
2241 if (IPLS->GenState == InProgressLookupState::InGenerator)
2242 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2243
2244 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2245 "Lookup should not be in InGenerator state here");
2246
2247 // FIXME: We should attach the query as we go: This provides a result in a
2248 // single pass in the common case where all symbols have already reached the
2249 // required state. The query could be detached again in the 'fail' method on
2250 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2251
2252 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2253
2254 // If we've been handed an error or received one back from a generator then
2255 // fail the query. We don't need to unlink: At this stage the query hasn't
2256 // actually been lodged.
2257 if (Err)
2258 return IPLS->fail(Err: std::move(Err));
2259
2260 // Get the next JITDylib and lookup flags.
2261 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2262 auto &JD = *KV.first;
2263 auto JDLookupFlags = KV.second;
2264
2265 LLVM_DEBUG({
2266 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2267 << ") with lookup set " << IPLS->LookupSet << ":\n";
2268 });
2269
2270 // If we've just reached a new JITDylib then perform some setup.
2271 if (IPLS->NewJITDylib) {
2272 // Add any non-candidates from the last JITDylib (if any) back on to the
2273 // list of definition candidates for this JITDylib, reset definition
2274 // non-candidates to the empty set.
2275 SymbolLookupSet Tmp;
2276 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2277 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2278
2279 LLVM_DEBUG({
2280 dbgs() << " First time visiting " << JD.getName()
2281 << ", resetting candidate sets and building generator stack\n";
2282 });
2283
2284 // Build the definition generator stack for this JITDylib.
2285 runSessionLocked(F: [&] {
2286 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2287 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2288 R: reverse(C&: JD.DefGenerators));
2289 });
2290
2291 // Flag that we've done our initialization.
2292 IPLS->NewJITDylib = false;
2293 }
2294
2295 // Remove any generation candidates that are already defined (and match) in
2296 // this JITDylib.
2297 runSessionLocked(F: [&] {
2298 // Update the list of candidates (and non-candidates) for definition
2299 // generation.
2300 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2301 Err = IL_updateCandidatesFor(
2302 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2303 NonCandidates: JD.DefGenerators.empty() ? nullptr
2304 : &IPLS->DefGeneratorNonCandidates);
2305 LLVM_DEBUG({
2306 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2307 << "\n";
2308 });
2309
2310 // If this lookup was resumed after auto-suspension but all candidates
2311 // have already been generated (by some previous call to the generator)
2312 // treat the lookup as if it had completed generation.
2313 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2314 IPLS->DefGeneratorCandidates.empty())
2315 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2316 });
2317
2318 // If we encountered an error while filtering generation candidates then
2319 // bail out.
2320 if (Err)
2321 return IPLS->fail(Err: std::move(Err));
2322
2323 /// Apply any definition generators on the stack.
2324 LLVM_DEBUG({
2325 if (IPLS->CurDefGeneratorStack.empty())
2326 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2327 else if (IPLS->DefGeneratorCandidates.empty())
2328 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2329 else
2330 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2331 << " remaining generators for "
2332 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2333 });
2334 while (!IPLS->CurDefGeneratorStack.empty() &&
2335 !IPLS->DefGeneratorCandidates.empty()) {
2336 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2337
2338 if (!DG)
2339 return IPLS->fail(Err: make_error<StringError>(
2340 Args: "DefinitionGenerator removed while lookup in progress",
2341 Args: inconvertibleErrorCode()));
2342
2343 // At this point the lookup is in either the NotInGenerator state, or in
2344 // the ResumedForGenerator state.
2345 // If this lookup is in the NotInGenerator state then check whether the
2346 // generator is in use. If the generator is not in use then move the
2347 // lookup to the InGenerator state and continue. If the generator is
2348 // already in use then just add this lookup to the pending lookups list
2349 // and bail out.
2350 // If this lookup is in the ResumedForGenerator state then just move it
2351 // to InGenerator and continue.
2352 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2353 std::lock_guard<std::mutex> Lock(DG->M);
2354 if (DG->InUse) {
2355 DG->PendingLookups.push_back(x: std::move(IPLS));
2356 return;
2357 }
2358 DG->InUse = true;
2359 }
2360
2361 IPLS->GenState = InProgressLookupState::InGenerator;
2362
2363 auto K = IPLS->K;
2364 auto &LookupSet = IPLS->DefGeneratorCandidates;
2365
2366 // Run the generator. If the generator takes ownership of QA then this
2367 // will break the loop.
2368 {
2369 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2370 LookupState LS(std::move(IPLS));
2371 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2372 IPLS = std::move(LS.IPLS);
2373 }
2374
2375 // If the lookup returned then pop the generator stack and unblock the
2376 // next lookup on this generator (if any).
2377 if (IPLS)
2378 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2379
2380 // If there was an error then fail the query.
2381 if (Err) {
2382 LLVM_DEBUG({
2383 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2384 });
2385 assert(IPLS && "LS cannot be retained if error is returned");
2386 return IPLS->fail(Err: std::move(Err));
2387 }
2388
2389 // Otherwise if QA was captured then break the loop.
2390 if (!IPLS) {
2391 LLVM_DEBUG(
2392 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2393 return;
2394 }
2395
2396 // Otherwise if we're continuing around the loop then update candidates
2397 // for the next round.
2398 runSessionLocked(F: [&] {
2399 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2400 Err = IL_updateCandidatesFor(
2401 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2402 NonCandidates: JD.DefGenerators.empty() ? nullptr
2403 : &IPLS->DefGeneratorNonCandidates);
2404 });
2405
2406 // If updating candidates failed then fail the query.
2407 if (Err) {
2408 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2409 return IPLS->fail(Err: std::move(Err));
2410 }
2411 }
2412
2413 if (IPLS->DefGeneratorCandidates.empty() &&
2414 IPLS->DefGeneratorNonCandidates.empty()) {
2415 // Early out if there are no remaining symbols.
2416 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2417 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2418 break;
2419 } else {
2420 // If we get here then we've moved on to the next JITDylib with candidates
2421 // remaining.
2422 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2423 ++IPLS->CurSearchOrderIndex;
2424 IPLS->NewJITDylib = true;
2425 }
2426 }
2427
2428 // Remove any weakly referenced candidates that could not be found/generated.
2429 IPLS->DefGeneratorCandidates.remove_if(
2430 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2431 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2432 });
2433
2434 // If we get here then we've finished searching all JITDylibs.
2435 // If we matched all symbols then move to phase 2, otherwise fail the query
2436 // with a SymbolsNotFound error.
2437 if (IPLS->DefGeneratorCandidates.empty()) {
2438 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2439 IPLS->complete(IPLS: std::move(IPLS));
2440 } else {
2441 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2442 IPLS->fail(Err: make_error<SymbolsNotFound>(
2443 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2444 }
2445}
2446
2447void ExecutionSession::OL_completeLookup(
2448 std::unique_ptr<InProgressLookupState> IPLS,
2449 std::shared_ptr<AsynchronousSymbolQuery> Q,
2450 RegisterDependenciesFunction RegisterDependencies) {
2451
2452 LLVM_DEBUG({
2453 dbgs() << "Entering OL_completeLookup:\n"
2454 << " Lookup kind: " << IPLS->K << "\n"
2455 << " Search order: " << IPLS->SearchOrder
2456 << ", Current index = " << IPLS->CurSearchOrderIndex
2457 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2458 << " Lookup set: " << IPLS->LookupSet << "\n"
2459 << " Definition generator candidates: "
2460 << IPLS->DefGeneratorCandidates << "\n"
2461 << " Definition generator non-candidates: "
2462 << IPLS->DefGeneratorNonCandidates << "\n";
2463 });
2464
2465 bool QueryComplete = false;
2466 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2467
2468 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2469 for (auto &KV : IPLS->SearchOrder) {
2470 auto &JD = *KV.first;
2471 auto JDLookupFlags = KV.second;
2472 LLVM_DEBUG({
2473 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2474 << ") with lookup set " << IPLS->LookupSet << ":\n";
2475 });
2476
2477 auto Err = IPLS->LookupSet.forEachWithRemoval(
2478 Body: [&](const SymbolStringPtr &Name,
2479 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2480 LLVM_DEBUG({
2481 dbgs() << " Attempting to match \"" << Name << "\" ("
2482 << SymLookupFlags << ")... ";
2483 });
2484
2485 /// Search for the symbol. If not found then continue without
2486 /// removal.
2487 auto SymI = JD.Symbols.find(Val: Name);
2488 if (SymI == JD.Symbols.end()) {
2489 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2490 return false;
2491 }
2492
2493 // If this is a non-exported symbol and we're matching exported
2494 // symbols only then skip this symbol without removal.
2495 if (!SymI->second.getFlags().isExported() &&
2496 JDLookupFlags ==
2497 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2498 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2499 return false;
2500 }
2501
2502 // If we match against a materialization-side-effects only symbol
2503 // then make sure it is weakly-referenced. Otherwise bail out with
2504 // an error.
2505 // FIXME: Use a "materialization-side-effects-only symbols must be
2506 // weakly referenced" specific error here to reduce confusion.
2507 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2508 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2509 LLVM_DEBUG({
2510 dbgs() << "error: "
2511 "required, but symbol is has-side-effects-only\n";
2512 });
2513 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2514 Args: SymbolNameVector({Name}));
2515 }
2516
2517 // If we matched against this symbol but it is in the error state
2518 // then bail out and treat it as a failure to materialize.
2519 if (SymI->second.getFlags().hasError()) {
2520 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2521 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2522 (*FailedSymbolsMap)[&JD] = {Name};
2523 return make_error<FailedToMaterialize>(
2524 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2525 }
2526
2527 // Otherwise this is a match.
2528
2529 // If this symbol is already in the required state then notify the
2530 // query, remove the symbol and continue.
2531 if (SymI->second.getState() >= Q->getRequiredState()) {
2532 LLVM_DEBUG(dbgs()
2533 << "matched, symbol already in required state\n");
2534 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2535
2536 // If this symbol is in anything other than the Ready state then
2537 // we need to track the dependence.
2538 if (SymI->second.getState() != SymbolState::Ready)
2539 Q->addQueryDependence(JD, Name);
2540
2541 return true;
2542 }
2543
2544 // Otherwise this symbol does not yet meet the required state. Check
2545 // whether it has a materializer attached, and if so prepare to run
2546 // it.
2547 if (SymI->second.hasMaterializerAttached()) {
2548 assert(SymI->second.getAddress() == ExecutorAddr() &&
2549 "Symbol not resolved but already has address?");
2550 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2551 assert(UMII != JD.UnmaterializedInfos.end() &&
2552 "Lazy symbol should have UnmaterializedInfo");
2553
2554 auto UMI = UMII->second;
2555 assert(UMI->MU && "Materializer should not be null");
2556 assert(UMI->RT && "Tracker should not be null");
2557 LLVM_DEBUG({
2558 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2559 << " (" << UMI->MU->getName() << ")\n";
2560 });
2561
2562 // Move all symbols associated with this MaterializationUnit into
2563 // materializing state.
2564 for (auto &KV : UMI->MU->getSymbols()) {
2565 auto SymK = JD.Symbols.find(Val: KV.first);
2566 assert(SymK != JD.Symbols.end() &&
2567 "No entry for symbol covered by MaterializationUnit");
2568 SymK->second.setMaterializerAttached(false);
2569 SymK->second.setState(SymbolState::Materializing);
2570 JD.UnmaterializedInfos.erase(Val: KV.first);
2571 }
2572
2573 // Add MU to the list of MaterializationUnits to be materialized.
2574 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2575 } else
2576 LLVM_DEBUG(dbgs() << "matched, registering query");
2577
2578 // Add the query to the PendingQueries list and continue, deleting
2579 // the element from the lookup set.
2580 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2581 SymI->second.getState() != SymbolState::Ready &&
2582 "By this line the symbol should be materializing");
2583 auto &MI = JD.MaterializingInfos[Name];
2584 MI.addQuery(Q);
2585 Q->addQueryDependence(JD, Name);
2586
2587 return true;
2588 });
2589
2590 JD.shrinkMaterializationInfoMemory();
2591
2592 // Handle failure.
2593 if (Err) {
2594
2595 LLVM_DEBUG({
2596 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2597 });
2598
2599 // Detach the query.
2600 Q->detach();
2601
2602 // Replace the MUs.
2603 for (auto &KV : CollectedUMIs) {
2604 auto &JD = *KV.first;
2605 for (auto &UMI : KV.second)
2606 for (auto &KV2 : UMI->MU->getSymbols()) {
2607 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2608 "Unexpected materializer in map");
2609 auto SymI = JD.Symbols.find(Val: KV2.first);
2610 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2611 assert(SymI->second.getState() == SymbolState::Materializing &&
2612 "Can not replace symbol that is not materializing");
2613 assert(!SymI->second.hasMaterializerAttached() &&
2614 "MaterializerAttached flag should not be set");
2615 SymI->second.setMaterializerAttached(true);
2616 JD.UnmaterializedInfos[KV2.first] = UMI;
2617 }
2618 }
2619
2620 return Err;
2621 }
2622 }
2623
2624 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2625 IPLS->LookupSet.forEachWithRemoval(
2626 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2627 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2628 Q->dropSymbol(Name);
2629 return true;
2630 } else
2631 return false;
2632 });
2633
2634 if (!IPLS->LookupSet.empty()) {
2635 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2636 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2637 Args: IPLS->LookupSet.getSymbolNames());
2638 }
2639
2640 // Record whether the query completed.
2641 QueryComplete = Q->isComplete();
2642
2643 LLVM_DEBUG({
2644 dbgs() << "Query successfully "
2645 << (QueryComplete ? "completed" : "lodged") << "\n";
2646 });
2647
2648 // Move the collected MUs to the OutstandingMUs list.
2649 if (!CollectedUMIs.empty()) {
2650 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2651
2652 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2653 for (auto &KV : CollectedUMIs) {
2654 LLVM_DEBUG({
2655 auto &JD = *KV.first;
2656 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2657 << " MUs.\n";
2658 });
2659 for (auto &UMI : KV.second) {
2660 auto MR = createMaterializationResponsibility(
2661 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2662 InitSymbol: std::move(UMI->MU->InitSymbol));
2663 OutstandingMUs.push_back(
2664 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2665 }
2666 }
2667 } else
2668 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2669
2670 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2671 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2672 RegisterDependencies(Q->QueryRegistrations);
2673 } else
2674 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2675
2676 return Error::success();
2677 });
2678
2679 if (LodgingErr) {
2680 LLVM_DEBUG(dbgs() << "Failing query\n");
2681 Q->detach();
2682 Q->handleFailed(Err: std::move(LodgingErr));
2683 return;
2684 }
2685
2686 if (QueryComplete) {
2687 LLVM_DEBUG(dbgs() << "Completing query\n");
2688 Q->handleComplete(ES&: *this);
2689 }
2690
2691 dispatchOutstandingMUs();
2692}
2693
2694void ExecutionSession::OL_completeLookupFlags(
2695 std::unique_ptr<InProgressLookupState> IPLS,
2696 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2697
2698 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2699 LLVM_DEBUG({
2700 dbgs() << "Entering OL_completeLookupFlags:\n"
2701 << " Lookup kind: " << IPLS->K << "\n"
2702 << " Search order: " << IPLS->SearchOrder
2703 << ", Current index = " << IPLS->CurSearchOrderIndex
2704 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2705 << " Lookup set: " << IPLS->LookupSet << "\n"
2706 << " Definition generator candidates: "
2707 << IPLS->DefGeneratorCandidates << "\n"
2708 << " Definition generator non-candidates: "
2709 << IPLS->DefGeneratorNonCandidates << "\n";
2710 });
2711
2712 SymbolFlagsMap Result;
2713
2714 // Attempt to find flags for each symbol.
2715 for (auto &KV : IPLS->SearchOrder) {
2716 auto &JD = *KV.first;
2717 auto JDLookupFlags = KV.second;
2718 LLVM_DEBUG({
2719 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2720 << ") with lookup set " << IPLS->LookupSet << ":\n";
2721 });
2722
2723 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2724 SymbolLookupFlags SymLookupFlags) {
2725 LLVM_DEBUG({
2726 dbgs() << " Attempting to match \"" << Name << "\" ("
2727 << SymLookupFlags << ")... ";
2728 });
2729
2730 // Search for the symbol. If not found then continue without removing
2731 // from the lookup set.
2732 auto SymI = JD.Symbols.find(Val: Name);
2733 if (SymI == JD.Symbols.end()) {
2734 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2735 return false;
2736 }
2737
2738 // If this is a non-exported symbol then it doesn't match. Skip it.
2739 if (!SymI->second.getFlags().isExported() &&
2740 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2741 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2742 return false;
2743 }
2744
2745 LLVM_DEBUG({
2746 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2747 << "\n";
2748 });
2749 Result[Name] = SymI->second.getFlags();
2750 return true;
2751 });
2752 }
2753
2754 // Remove any weakly referenced symbols that haven't been resolved.
2755 IPLS->LookupSet.remove_if(
2756 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2757 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2758 });
2759
2760 if (!IPLS->LookupSet.empty()) {
2761 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2762 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2763 Args: IPLS->LookupSet.getSymbolNames());
2764 }
2765
2766 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2767 return Result;
2768 });
2769
2770 // Run the callback on the result.
2771 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2772 OnComplete(std::move(Result));
2773}
2774
2775void ExecutionSession::OL_destroyMaterializationResponsibility(
2776 MaterializationResponsibility &MR) {
2777
2778 assert(MR.SymbolFlags.empty() &&
2779 "All symbols should have been explicitly materialized or failed");
2780 MR.JD.unlinkMaterializationResponsibility(MR);
2781}
2782
2783SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2784 const MaterializationResponsibility &MR) {
2785 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2786}
2787
2788Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2789 const SymbolMap &Symbols) {
2790 LLVM_DEBUG({
2791 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2792 });
2793#ifndef NDEBUG
2794 for (auto &KV : Symbols) {
2795 auto I = MR.SymbolFlags.find(KV.first);
2796 assert(I != MR.SymbolFlags.end() &&
2797 "Resolving symbol outside this responsibility set");
2798 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2799 "Can't resolve materialization-side-effects-only symbol");
2800 if (I->second & JITSymbolFlags::Common) {
2801 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2802 assert((KV.second.getFlags() & WeakOrCommon) &&
2803 "Common symbols must be resolved as common or weak");
2804 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2805 (I->second & ~JITSymbolFlags::Common) &&
2806 "Resolving symbol with incorrect flags");
2807 } else
2808 assert(KV.second.getFlags() == I->second &&
2809 "Resolving symbol with incorrect flags");
2810 }
2811#endif
2812
2813 return MR.JD.resolve(MR, Resolved: Symbols);
2814}
2815
2816WaitingOnGraph::ExternalState
2817ExecutionSession::IL_getSymbolState(JITDylib *JD,
2818 NonOwningSymbolStringPtr Name) {
2819 if (JD->State != JITDylib::Open)
2820 return WaitingOnGraph::ExternalState::Failed;
2821
2822 auto I = JD->Symbols.find_as(Val: Name);
2823
2824 // FIXME: Can we eliminate this possibility if we support query binding?
2825 if (I == JD->Symbols.end())
2826 return WaitingOnGraph::ExternalState::Failed;
2827
2828 if (I->second.getFlags().hasError())
2829 return WaitingOnGraph::ExternalState::Failed;
2830
2831 if (I->second.getState() == SymbolState::Ready)
2832 return WaitingOnGraph::ExternalState::Ready;
2833
2834 return WaitingOnGraph::ExternalState::None;
2835}
2836
2837template <typename UpdateSymbolFn, typename UpdateQueryFn>
2838void ExecutionSession::IL_collectQueries(
2839 JITDylib::AsynchronousSymbolQuerySet &Qs,
2840 WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
2841 UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
2842
2843 for (auto &[JD, Symbols] : QualifiedSymbols) {
2844 // IL_emit and JITDylib removal are synchronized by the session lock.
2845 // Since JITDylib removal removes any contained nodes from the
2846 // WaitingOnGraph, we should be able to assert that all nodes in the
2847 // WaitingOnGraph have not been removed.
2848 assert(JD->State == JITDylib::Open &&
2849 "WaitingOnGraph includes definition in defunct JITDylib");
2850 for (auto &Symbol : Symbols) {
2851 // Update symbol table.
2852 auto I = JD->Symbols.find_as(Val: Symbol);
2853 assert(I != JD->Symbols.end() &&
2854 "Failed Symbol missing from JD symbol table");
2855 auto &Entry = I->second;
2856 UpdateSymbol(Entry);
2857
2858 // Collect queries.
2859 auto J = JD->MaterializingInfos.find_as(Val: Symbol);
2860 if (J != JD->MaterializingInfos.end()) {
2861 for (auto &Q : J->second.takeAllPendingQueries()) {
2862 UpdateQuery(*Q, *JD, Symbol, Entry);
2863 Qs.insert(x: std::move(Q));
2864 }
2865 JD->MaterializingInfos.erase(I: J);
2866 }
2867 }
2868 }
2869}
2870
2871Expected<ExecutionSession::EmitQueries>
2872ExecutionSession::IL_emit(MaterializationResponsibility &MR,
2873 WaitingOnGraph::SimplifyResult SR) {
2874
2875 if (MR.RT->isDefunct())
2876 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
2877
2878 auto &TargetJD = MR.getTargetJITDylib();
2879 if (TargetJD.State != JITDylib::Open)
2880 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
2881 " is defunct",
2882 Args: inconvertibleErrorCode());
2883
2884#ifdef EXPENSIVE_CHECKS
2885 verifySessionState("entering ExecutionSession::IL_emit");
2886#endif
2887
2888 auto ER = G.emit(SR: std::move(SR),
2889 GetExternalState: [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
2890 return IL_getSymbolState(JD, Name);
2891 });
2892
2893 EmitQueries EQ;
2894
2895 // Handle failed queries.
2896 for (auto &SN : ER.Failed)
2897 IL_collectQueries(
2898 Qs&: EQ.Failed, QualifiedSymbols&: SN->defs(),
2899 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) {
2900 E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
2901 },
2902 UpdateQuery: [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
2903 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2904 auto &FS = EQ.FailedSymsForQuery[&Q];
2905 if (!FS)
2906 FS = std::make_shared<SymbolDependenceMap>();
2907 (*FS)[&JD].insert(V: SymbolStringPtr(Name));
2908 });
2909
2910 for (auto &FQ : EQ.Failed)
2911 FQ->detach();
2912
2913 for (auto &SN : ER.Ready)
2914 IL_collectQueries(
2915 Qs&: EQ.Completed, QualifiedSymbols&: SN->defs(),
2916 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
2917 UpdateQuery: [](AsynchronousSymbolQuery &Q, JITDylib &JD,
2918 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2919 Q.notifySymbolMetRequiredState(Name: SymbolStringPtr(Name), Sym: E.getSymbol());
2920 });
2921
2922 // std::erase_if is not available in C++17, and llvm::erase_if does not work
2923 // here.
2924 for (auto it = EQ.Completed.begin(), end = EQ.Completed.end(); it != end;) {
2925 if ((*it)->isComplete()) {
2926 ++it;
2927 } else {
2928 it = EQ.Completed.erase(position: it);
2929 }
2930 }
2931
2932#ifdef EXPENSIVE_CHECKS
2933 verifySessionState("exiting ExecutionSession::IL_emit");
2934#endif
2935
2936 return std::move(EQ);
2937}
2938
2939Error ExecutionSession::OL_notifyEmitted(
2940 MaterializationResponsibility &MR,
2941 ArrayRef<SymbolDependenceGroup> DepGroups) {
2942 LLVM_DEBUG({
2943 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
2944 << "\n";
2945 if (!DepGroups.empty()) {
2946 dbgs() << " Initial dependencies:\n";
2947 for (auto &SDG : DepGroups) {
2948 dbgs() << " Symbols: " << SDG.Symbols
2949 << ", Dependencies: " << SDG.Dependencies << "\n";
2950 }
2951 }
2952 });
2953
2954#ifndef NDEBUG
2955 SymbolNameSet Visited;
2956 for (auto &DG : DepGroups) {
2957 for (auto &Sym : DG.Symbols) {
2958 assert(MR.SymbolFlags.count(Sym) &&
2959 "DG contains dependence for symbol outside this MR");
2960 assert(Visited.insert(Sym).second &&
2961 "DG contains duplicate entries for Name");
2962 }
2963 }
2964#endif // NDEBUG
2965
2966 std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
2967 WaitingOnGraph::ContainerElementsMap Residual;
2968 {
2969 auto &JDResidual = Residual[&MR.getTargetJITDylib()];
2970 for (auto &[Name, Flags] : MR.getSymbols())
2971 JDResidual.insert(V: NonOwningSymbolStringPtr(Name));
2972
2973 for (auto &SDG : DepGroups) {
2974 WaitingOnGraph::ContainerElementsMap Defs;
2975 assert(!SDG.Symbols.empty());
2976 auto &JDDefs = Defs[&MR.getTargetJITDylib()];
2977 for (auto &Def : SDG.Symbols) {
2978 JDDefs.insert(V: NonOwningSymbolStringPtr(Def));
2979 JDResidual.erase(V: NonOwningSymbolStringPtr(Def));
2980 }
2981 WaitingOnGraph::ContainerElementsMap Deps;
2982 if (!SDG.Dependencies.empty()) {
2983 for (auto &[JD, Syms] : SDG.Dependencies) {
2984 auto &JDDeps = Deps[JD];
2985 for (auto &Dep : Syms)
2986 JDDeps.insert(V: NonOwningSymbolStringPtr(Dep));
2987 }
2988 }
2989 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2990 args: std::move(Defs), args: std::move(Deps)));
2991 }
2992 if (!JDResidual.empty())
2993 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2994 args: std::move(Residual), args: WaitingOnGraph::ContainerElementsMap()));
2995 }
2996
2997 auto SR = WaitingOnGraph::simplify(SNs: std::move(SNs), Rec: GOpRecorder);
2998
2999 LLVM_DEBUG({
3000 dbgs() << " Simplified dependencies:\n";
3001 for (auto &SN : SR.superNodes()) {
3002
3003 auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
3004 std::vector<JITDylib *> JDs;
3005 for (auto &[JD, _] : C)
3006 JDs.push_back(JD);
3007 llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
3008 return LHS->getName() < RHS->getName();
3009 });
3010 return JDs;
3011 };
3012
3013 auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
3014 std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
3015 llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
3016 const NonOwningSymbolStringPtr &RHS) {
3017 return *LHS < *RHS;
3018 });
3019 return Names;
3020 };
3021
3022 dbgs() << " Defs: {";
3023 for (auto *JD : SortedLibs(SN->defs())) {
3024 dbgs() << " (" << JD->getName() << ", [";
3025 for (auto &Sym : SortedNames(SN->defs()[JD]))
3026 dbgs() << " " << Sym;
3027 dbgs() << " ])";
3028 }
3029 dbgs() << " }, Deps: {";
3030 for (auto *JD : SortedLibs(SN->deps())) {
3031 dbgs() << " (" << JD->getName() << ", [";
3032 for (auto &Sym : SortedNames(SN->deps()[JD]))
3033 dbgs() << " " << Sym;
3034 dbgs() << " ])";
3035 }
3036 dbgs() << " }\n";
3037 }
3038 });
3039 auto EmitQueries =
3040 runSessionLocked(F: [&]() { return IL_emit(MR, SR: std::move(SR)); });
3041
3042 // On error bail out.
3043 if (!EmitQueries)
3044 return EmitQueries.takeError();
3045
3046 // Otherwise notify failed queries, and any updated queries that have been
3047 // completed.
3048
3049 // FIXME: Get rid of error return from notifyEmitted.
3050 SymbolDependenceMap BadDeps;
3051 {
3052 for (auto &FQ : EmitQueries->Failed) {
3053 FQ->detach();
3054 assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
3055 "Missing failed symbols for query");
3056 auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
3057 for (auto &[JD, Syms] : *FailedSyms) {
3058 auto &BadDepsForJD = BadDeps[JD];
3059 for (auto &Sym : Syms)
3060 BadDepsForJD.insert(V: Sym);
3061 }
3062 FQ->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
3063 Args: std::move(FailedSyms)));
3064 }
3065 }
3066
3067 for (auto &UQ : EmitQueries->Completed)
3068 UQ->handleComplete(ES&: *this);
3069
3070 // If there are any bad dependencies then return an error.
3071 if (!BadDeps.empty()) {
3072 SymbolNameSet BadNames;
3073 // Note: The name set calculated here is bogus: it includes all symbols in
3074 // the MR, not just the ones that failed. We want to remove the error
3075 // return path from notifyEmitted anyway, so this is just a brief
3076 // placeholder to maintain (roughly) the current error behavior.
3077 for (auto &[Name, Flags] : MR.getSymbols())
3078 BadNames.insert(V: Name);
3079 MR.SymbolFlags.clear();
3080 return make_error<UnsatisfiedSymbolDependencies>(
3081 Args: getSymbolStringPool(), Args: &MR.getTargetJITDylib(), Args: std::move(BadNames),
3082 Args: std::move(BadDeps), Args: "dependencies removed or in error state");
3083 }
3084
3085 MR.SymbolFlags.clear();
3086 return Error::success();
3087}
3088
3089Error ExecutionSession::OL_defineMaterializing(
3090 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3091
3092 LLVM_DEBUG({
3093 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3094 << NewSymbolFlags << "\n";
3095 });
3096 if (auto AcceptedDefs =
3097 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3098 // Add all newly accepted symbols to this responsibility object.
3099 for (auto &KV : *AcceptedDefs)
3100 MR.SymbolFlags.insert(KV);
3101 return Error::success();
3102 } else
3103 return AcceptedDefs.takeError();
3104}
3105
3106std::pair<JITDylib::AsynchronousSymbolQuerySet,
3107 std::shared_ptr<SymbolDependenceMap>>
3108ExecutionSession::IL_failSymbols(JITDylib &JD,
3109 const SymbolNameVector &SymbolsToFail) {
3110
3111#ifdef EXPENSIVE_CHECKS
3112 verifySessionState("entering ExecutionSession::IL_failSymbols");
3113#endif
3114
3115 // Early out in the easy case.
3116 if (SymbolsToFail.empty())
3117 return {};
3118
3119 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3120 auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
3121 auto I = FailJD->Symbols.find_as(Val: FailSym);
3122 assert(I != FailJD->Symbols.end());
3123 I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
3124 auto J = FailJD->MaterializingInfos.find_as(Val: FailSym);
3125 if (J != FailJD->MaterializingInfos.end()) {
3126 for (auto &Q : J->second.takeAllPendingQueries())
3127 FailedQueries.insert(x: std::move(Q));
3128 FailJD->MaterializingInfos.erase(I: J);
3129 }
3130 };
3131
3132 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3133
3134 {
3135 auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
3136 for (auto &Sym : SymbolsToFail) {
3137 FailedSymsForJD.insert(V: Sym);
3138 Fail(&JD, NonOwningSymbolStringPtr(Sym));
3139 }
3140 }
3141
3142 WaitingOnGraph::ContainerElementsMap ToFail;
3143 auto &JDToFail = ToFail[&JD];
3144 for (auto &Sym : SymbolsToFail)
3145 JDToFail.insert(V: NonOwningSymbolStringPtr(Sym));
3146
3147 auto FailedSNs = G.fail(Failed: ToFail, Rec: GOpRecorder);
3148
3149 for (auto &SN : FailedSNs) {
3150 for (auto &[FailJD, Defs] : SN->defs()) {
3151 auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
3152 for (auto &Def : Defs) {
3153 FailedSymsForFailJD.insert(V: SymbolStringPtr(Def));
3154 Fail(FailJD, Def);
3155 }
3156 }
3157 }
3158
3159 // Detach all failed queries.
3160 for (auto &Q : FailedQueries)
3161 Q->detach();
3162
3163#ifdef EXPENSIVE_CHECKS
3164 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3165#endif
3166
3167 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3168}
3169
3170void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3171
3172 LLVM_DEBUG({
3173 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3174 << MR.SymbolFlags << "\n";
3175 });
3176
3177 if (MR.SymbolFlags.empty())
3178 return;
3179
3180 SymbolNameVector SymbolsToFail;
3181 for (auto &[Name, Flags] : MR.SymbolFlags)
3182 SymbolsToFail.push_back(x: Name);
3183 MR.SymbolFlags.clear();
3184
3185 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3186 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3187
3188 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3189 // If the tracker is defunct then there's nothing to do here.
3190 if (MR.RT->isDefunct())
3191 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3192 std::shared_ptr<SymbolDependenceMap>>();
3193 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3194 });
3195
3196 for (auto &Q : FailedQueries) {
3197 Q->detach();
3198 Q->handleFailed(
3199 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3200 }
3201}
3202
3203Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3204 std::unique_ptr<MaterializationUnit> MU) {
3205 for (auto &KV : MU->getSymbols()) {
3206 assert(MR.SymbolFlags.count(KV.first) &&
3207 "Replacing definition outside this responsibility set");
3208 MR.SymbolFlags.erase(Val: KV.first);
3209 }
3210
3211 if (MU->getInitializerSymbol() == MR.InitSymbol)
3212 MR.InitSymbol = nullptr;
3213
3214 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3215 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3216 << "\n";
3217 }););
3218
3219 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3220}
3221
3222Expected<std::unique_ptr<MaterializationResponsibility>>
3223ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3224 const SymbolNameSet &Symbols) {
3225
3226 SymbolStringPtr DelegatedInitSymbol;
3227 SymbolFlagsMap DelegatedFlags;
3228
3229 for (auto &Name : Symbols) {
3230 auto I = MR.SymbolFlags.find(Val: Name);
3231 assert(I != MR.SymbolFlags.end() &&
3232 "Symbol is not tracked by this MaterializationResponsibility "
3233 "instance");
3234
3235 DelegatedFlags[Name] = std::move(I->second);
3236 if (Name == MR.InitSymbol)
3237 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3238
3239 MR.SymbolFlags.erase(I);
3240 }
3241
3242 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3243 InitSymbol: std::move(DelegatedInitSymbol));
3244}
3245
3246#ifndef NDEBUG
3247void ExecutionSession::dumpDispatchInfo(Task &T) {
3248 runSessionLocked([&]() {
3249 dbgs() << "Dispatching: ";
3250 T.printDescription(dbgs());
3251 dbgs() << "\n";
3252 });
3253}
3254#endif // NDEBUG
3255
3256} // End namespace orc.
3257} // End namespace llvm.
3258