1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
14#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
16#include "llvm/Support/FormatVariadic.h"
17#include "llvm/Support/MSVCErrorWorkarounds.h"
18#include "llvm/Support/raw_ostream.h"
19
20#include <condition_variable>
21#include <future>
22#include <optional>
23
24#define DEBUG_TYPE "orc"
25
26namespace llvm {
27namespace orc {
28
29char ResourceTrackerDefunct::ID = 0;
30char JITDylibDefunct::ID = 0;
31char FailedToMaterialize::ID = 0;
32char SymbolsNotFound::ID = 0;
33char SymbolsCouldNotBeRemoved::ID = 0;
34char MissingSymbolDefinitions::ID = 0;
35char UnexpectedSymbolDefinitions::ID = 0;
36char UnsatisfiedSymbolDependencies::ID = 0;
37char MaterializationTask::ID = 0;
38char LookupTask::ID = 0;
39
40RegisterDependenciesFunction NoDependenciesToRegister =
41 RegisterDependenciesFunction();
42
43void MaterializationUnit::anchor() {}
44
45ResourceTracker::ResourceTracker(JITDylibSP JD) {
46 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
47 "JITDylib must be two byte aligned");
48 JD->Retain();
49 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
50}
51
52ResourceTracker::~ResourceTracker() {
53 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
54 getJITDylib().Release();
55}
56
57Error ResourceTracker::remove() {
58 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
59}
60
61void ResourceTracker::transferTo(ResourceTracker &DstRT) {
62 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
63}
64
65void ResourceTracker::makeDefunct() {
66 uintptr_t Val = JDAndFlag.load();
67 Val |= 0x1U;
68 JDAndFlag.store(i: Val);
69}
70
71ResourceManager::~ResourceManager() = default;
72
73ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
74 : RT(std::move(RT)) {}
75
76std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
77 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
78}
79
80void ResourceTrackerDefunct::log(raw_ostream &OS) const {
81 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
82}
83
84std::error_code JITDylibDefunct::convertToErrorCode() const {
85 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
86}
87
88void JITDylibDefunct::log(raw_ostream &OS) const {
89 OS << "JITDylib " << JD->getName() << " (" << (void *)JD.get()
90 << ") is defunct";
91}
92
93FailedToMaterialize::FailedToMaterialize(
94 std::shared_ptr<SymbolStringPool> SSP,
95 std::shared_ptr<SymbolDependenceMap> Symbols)
96 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
97 assert(this->SSP && "String pool cannot be null");
98 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
99
100 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
101 // don't have to manually retain/release.
102 for (auto &[JD, Syms] : *this->Symbols)
103 JD->Retain();
104}
105
106FailedToMaterialize::~FailedToMaterialize() {
107 for (auto &[JD, Syms] : *Symbols)
108 JD->Release();
109}
110
111std::error_code FailedToMaterialize::convertToErrorCode() const {
112 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
113}
114
115void FailedToMaterialize::log(raw_ostream &OS) const {
116 OS << "Failed to materialize symbols: " << *Symbols;
117}
118
119UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
120 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
121 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
122 std::string Explanation)
123 : SSP(std::move(SSP)), JD(std::move(JD)),
124 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
125 Explanation(std::move(Explanation)) {}
126
127std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
128 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
129}
130
131void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
132 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
133 << ", due to unsatisfied dependencies " << BadDeps;
134 if (!Explanation.empty())
135 OS << " (" << Explanation << ")";
136}
137
138SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
139 SymbolNameSet Symbols)
140 : SSP(std::move(SSP)) {
141 llvm::append_range(C&: this->Symbols, R&: Symbols);
142 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
143}
144
145SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
146 SymbolNameVector Symbols)
147 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
148 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
149}
150
151std::error_code SymbolsNotFound::convertToErrorCode() const {
152 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
153}
154
155void SymbolsNotFound::log(raw_ostream &OS) const {
156 OS << "Symbols not found: " << Symbols;
157}
158
159SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
160 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
161 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
162 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
163}
164
165std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
166 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
167}
168
169void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
170 OS << "Symbols could not be removed: " << Symbols;
171}
172
173std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
174 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
175}
176
177void MissingSymbolDefinitions::log(raw_ostream &OS) const {
178 OS << "Missing definitions in module " << ModuleName
179 << ": " << Symbols;
180}
181
182std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
183 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
184}
185
186void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
187 OS << "Unexpected definitions in module " << ModuleName
188 << ": " << Symbols;
189}
190
191void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
192 JD->getExecutionSession().lookup(
193 K: LookupKind::Static, SearchOrder: {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
194 Symbols: SymbolLookupSet(Name), RequiredState: SymbolState::Ready,
195 NotifyComplete: [OnComplete = std::move(OnComplete)
196#ifndef NDEBUG
197 ,
198 Name = this->Name // Captured for the assert below only.
199#endif // NDEBUG
200 ](Expected<SymbolMap> Result) mutable {
201 if (Result) {
202 assert(Result->size() == 1 && "Unexpected number of results");
203 assert(Result->count(Name) &&
204 "Result does not contain expected symbol");
205 OnComplete(Result->begin()->second);
206 } else
207 OnComplete(Result.takeError());
208 },
209 RegisterDependencies: NoDependenciesToRegister);
210}
211
212AsynchronousSymbolQuery::AsynchronousSymbolQuery(
213 const SymbolLookupSet &Symbols, SymbolState RequiredState,
214 SymbolsResolvedCallback NotifyComplete)
215 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
216 assert(RequiredState >= SymbolState::Resolved &&
217 "Cannot query for a symbols that have not reached the resolve state "
218 "yet");
219
220 OutstandingSymbolsCount = Symbols.size();
221
222 for (auto &[Name, Flags] : Symbols)
223 ResolvedSymbols[Name] = ExecutorSymbolDef();
224}
225
226void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
227 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
228 auto I = ResolvedSymbols.find(Val: Name);
229 assert(I != ResolvedSymbols.end() &&
230 "Resolving symbol outside the requested set");
231 assert(I->second == ExecutorSymbolDef() &&
232 "Redundantly resolving symbol Name");
233
234 // If this is a materialization-side-effects-only symbol then drop it,
235 // otherwise update its map entry with its resolved address.
236 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
237 ResolvedSymbols.erase(I);
238 else
239 I->second = std::move(Sym);
240 --OutstandingSymbolsCount;
241}
242
243void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
244 assert(OutstandingSymbolsCount == 0 &&
245 "Symbols remain, handleComplete called prematurely");
246
247 class RunQueryCompleteTask : public Task {
248 public:
249 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
250 SymbolsResolvedCallback NotifyComplete)
251 : ResolvedSymbols(std::move(ResolvedSymbols)),
252 NotifyComplete(std::move(NotifyComplete)) {}
253 void printDescription(raw_ostream &OS) override {
254 OS << "Execute query complete callback for " << ResolvedSymbols;
255 }
256 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
257
258 private:
259 SymbolMap ResolvedSymbols;
260 SymbolsResolvedCallback NotifyComplete;
261 };
262
263 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
264 args: std::move(NotifyComplete));
265 NotifyComplete = SymbolsResolvedCallback();
266 ES.dispatchTask(T: std::move(T));
267}
268
269void AsynchronousSymbolQuery::handleFailed(Error Err) {
270 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
271 OutstandingSymbolsCount == 0 &&
272 "Query should already have been abandoned");
273 NotifyComplete(std::move(Err));
274 NotifyComplete = SymbolsResolvedCallback();
275}
276
277void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
278 SymbolStringPtr Name) {
279 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
280 (void)Added;
281 assert(Added && "Duplicate dependence notification?");
282}
283
284void AsynchronousSymbolQuery::removeQueryDependence(
285 JITDylib &JD, const SymbolStringPtr &Name) {
286 auto QRI = QueryRegistrations.find(Val: &JD);
287 assert(QRI != QueryRegistrations.end() &&
288 "No dependencies registered for JD");
289 assert(QRI->second.count(Name) && "No dependency on Name in JD");
290 QRI->second.erase(V: Name);
291 if (QRI->second.empty())
292 QueryRegistrations.erase(I: QRI);
293}
294
295void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
296 auto I = ResolvedSymbols.find(Val: Name);
297 assert(I != ResolvedSymbols.end() &&
298 "Redundant removal of weakly-referenced symbol");
299 ResolvedSymbols.erase(I);
300 --OutstandingSymbolsCount;
301}
302
303void AsynchronousSymbolQuery::detach() {
304 ResolvedSymbols.clear();
305 OutstandingSymbolsCount = 0;
306 for (auto &[JD, Syms] : QueryRegistrations)
307 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
308 QueryRegistrations.clear();
309}
310
311ReExportsMaterializationUnit::ReExportsMaterializationUnit(
312 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
313 SymbolAliasMap Aliases)
314 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
315 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
316
317StringRef ReExportsMaterializationUnit::getName() const {
318 return "<Reexports>";
319}
320
321void ReExportsMaterializationUnit::materialize(
322 std::unique_ptr<MaterializationResponsibility> R) {
323
324 auto &ES = R->getTargetJITDylib().getExecutionSession();
325 JITDylib &TgtJD = R->getTargetJITDylib();
326 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
327
328 // Find the set of requested aliases and aliasees. Return any unrequested
329 // aliases back to the JITDylib so as to not prematurely materialize any
330 // aliasees.
331 auto RequestedSymbols = R->getRequestedSymbols();
332 SymbolAliasMap RequestedAliases;
333
334 for (auto &Name : RequestedSymbols) {
335 auto I = Aliases.find(Val: Name);
336 assert(I != Aliases.end() && "Symbol not found in aliases map?");
337 RequestedAliases[Name] = std::move(I->second);
338 Aliases.erase(I);
339 }
340
341 LLVM_DEBUG({
342 ES.runSessionLocked([&]() {
343 dbgs() << "materializing reexports: target = " << TgtJD.getName()
344 << ", source = " << SrcJD.getName() << " " << RequestedAliases
345 << "\n";
346 });
347 });
348
349 if (!Aliases.empty()) {
350 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
351 SourceJDLookupFlags))
352 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
353
354 if (Err) {
355 // FIXME: Should this be reported / treated as failure to materialize?
356 // Or should this be treated as a sanctioned bailing-out?
357 ES.reportError(Err: std::move(Err));
358 R->failMaterialization();
359 return;
360 }
361 }
362
363 // The OnResolveInfo struct will hold the aliases and responsibility for each
364 // query in the list.
365 struct OnResolveInfo {
366 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
367 SymbolAliasMap Aliases)
368 : R(std::move(R)), Aliases(std::move(Aliases)) {}
369
370 std::unique_ptr<MaterializationResponsibility> R;
371 SymbolAliasMap Aliases;
372 std::vector<SymbolDependenceGroup> SDGs;
373 };
374
375 // Build a list of queries to issue. In each round we build a query for the
376 // largest set of aliases that we can resolve without encountering a chain of
377 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
378 // query would be waiting on a symbol that it itself had to resolve. Creating
379 // a new query for each link in such a chain eliminates the possibility of
380 // deadlock. In practice chains are likely to be rare, and this algorithm will
381 // usually result in a single query to issue.
382
383 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
384 QueryInfos;
385 while (!RequestedAliases.empty()) {
386 SymbolNameSet ResponsibilitySymbols;
387 SymbolLookupSet QuerySymbols;
388 SymbolAliasMap QueryAliases;
389
390 // Collect as many aliases as we can without including a chain.
391 for (auto &[Alias, AliasInfo] : RequestedAliases) {
392 // Chain detected. Skip this symbol for this round.
393 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: AliasInfo.Aliasee) ||
394 RequestedAliases.count(Val: AliasInfo.Aliasee)))
395 continue;
396
397 ResponsibilitySymbols.insert(V: Alias);
398 QuerySymbols.add(Name: AliasInfo.Aliasee,
399 Flags: AliasInfo.AliasFlags.hasMaterializationSideEffectsOnly()
400 ? SymbolLookupFlags::WeaklyReferencedSymbol
401 : SymbolLookupFlags::RequiredSymbol);
402 QueryAliases[Alias] = std::move(AliasInfo);
403 }
404
405 // Remove the aliases collected this round from the RequestedAliases map.
406 for (auto &KV : QueryAliases)
407 RequestedAliases.erase(Val: KV.first);
408
409 assert(!QuerySymbols.empty() && "Alias cycle detected!");
410
411 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
412 if (!NewR) {
413 ES.reportError(Err: NewR.takeError());
414 R->failMaterialization();
415 return;
416 }
417
418 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
419 args: std::move(QueryAliases));
420 QueryInfos.push_back(
421 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
422 }
423
424 // Issue the queries.
425 while (!QueryInfos.empty()) {
426 auto QuerySymbols = std::move(QueryInfos.back().first);
427 auto QueryInfo = std::move(QueryInfos.back().second);
428
429 QueryInfos.pop_back();
430
431 auto RegisterDependencies = [QueryInfo,
432 &SrcJD](const SymbolDependenceMap &Deps) {
433 // If there were no materializing symbols, just bail out.
434 if (Deps.empty())
435 return;
436
437 // Otherwise the only deps should be on SrcJD.
438 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
439 "Unexpected dependencies for reexports");
440
441 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
442
443 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
444 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
445 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
446 };
447
448 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
449 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
450 if (Result) {
451 SymbolMap ResolutionMap;
452 for (auto &KV : QueryInfo->Aliases) {
453 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
454 Result->count(KV.second.Aliasee)) &&
455 "Result map missing entry?");
456 // Don't try to resolve materialization-side-effects-only symbols.
457 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
458 continue;
459
460 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
461 KV.second.AliasFlags};
462 }
463 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
464 ES.reportError(Err: std::move(Err));
465 QueryInfo->R->failMaterialization();
466 return;
467 }
468 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
469 ES.reportError(Err: std::move(Err));
470 QueryInfo->R->failMaterialization();
471 return;
472 }
473 } else {
474 ES.reportError(Err: Result.takeError());
475 QueryInfo->R->failMaterialization();
476 }
477 };
478
479 ES.lookup(K: LookupKind::Static,
480 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
481 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
482 RegisterDependencies: std::move(RegisterDependencies));
483 }
484}
485
486void ReExportsMaterializationUnit::discard(const JITDylib &JD,
487 const SymbolStringPtr &Name) {
488 assert(Aliases.count(Name) &&
489 "Symbol not covered by this MaterializationUnit");
490 Aliases.erase(Val: Name);
491}
492
493MaterializationUnit::Interface
494ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
495 SymbolFlagsMap SymbolFlags;
496 for (auto &KV : Aliases)
497 SymbolFlags[KV.first] = KV.second.AliasFlags;
498
499 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
500}
501
502Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
503 SymbolNameSet Symbols) {
504 SymbolLookupSet LookupSet(Symbols);
505 auto Flags = SourceJD.getExecutionSession().lookupFlags(
506 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
507 Symbols: SymbolLookupSet(std::move(Symbols)));
508
509 if (!Flags)
510 return Flags.takeError();
511
512 SymbolAliasMap Result;
513 for (auto &Name : Symbols) {
514 assert(Flags->count(Name) && "Missing entry in flags map");
515 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
516 }
517
518 return Result;
519}
520
521class InProgressLookupState {
522public:
523 // FIXME: Reduce the number of SymbolStringPtrs here. See
524 // https://github.com/llvm/llvm-project/issues/55576.
525
526 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
527 SymbolLookupSet LookupSet, SymbolState RequiredState)
528 : K(K), SearchOrder(std::move(SearchOrder)),
529 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
530 DefGeneratorCandidates = this->LookupSet;
531 }
532 virtual ~InProgressLookupState() = default;
533 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
534 virtual void fail(Error Err) = 0;
535
536 LookupKind K;
537 JITDylibSearchOrder SearchOrder;
538 SymbolLookupSet LookupSet;
539 SymbolState RequiredState;
540
541 size_t CurSearchOrderIndex = 0;
542 bool NewJITDylib = true;
543 SymbolLookupSet DefGeneratorCandidates;
544 SymbolLookupSet DefGeneratorNonCandidates;
545
546 enum {
547 NotInGenerator, // Not currently using a generator.
548 ResumedForGenerator, // Resumed after being auto-suspended before generator.
549 InGenerator // Currently using generator.
550 } GenState = NotInGenerator;
551 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
552};
553
554class InProgressLookupFlagsState : public InProgressLookupState {
555public:
556 InProgressLookupFlagsState(
557 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
558 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
559 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
560 SymbolState::NeverSearched),
561 OnComplete(std::move(OnComplete)) {}
562
563 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
564 auto &ES = SearchOrder.front().first->getExecutionSession();
565 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
566 }
567
568 void fail(Error Err) override { OnComplete(std::move(Err)); }
569
570private:
571 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
572};
573
574class InProgressFullLookupState : public InProgressLookupState {
575public:
576 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
577 SymbolLookupSet LookupSet,
578 SymbolState RequiredState,
579 std::shared_ptr<AsynchronousSymbolQuery> Q,
580 RegisterDependenciesFunction RegisterDependencies)
581 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
582 RequiredState),
583 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
584 }
585
586 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
587 auto &ES = SearchOrder.front().first->getExecutionSession();
588 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
589 RegisterDependencies: std::move(RegisterDependencies));
590 }
591
592 void fail(Error Err) override {
593 Q->detach();
594 Q->handleFailed(Err: std::move(Err));
595 }
596
597private:
598 std::shared_ptr<AsynchronousSymbolQuery> Q;
599 RegisterDependenciesFunction RegisterDependencies;
600};
601
602ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
603 JITDylibLookupFlags SourceJDLookupFlags,
604 SymbolPredicate Allow)
605 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
606 Allow(std::move(Allow)) {}
607
608Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
609 JITDylib &JD,
610 JITDylibLookupFlags JDLookupFlags,
611 const SymbolLookupSet &LookupSet) {
612 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
613
614 // Use lookupFlags to find the subset of symbols that match our lookup.
615 auto Flags = JD.getExecutionSession().lookupFlags(
616 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
617 if (!Flags)
618 return Flags.takeError();
619
620 // Create an alias map.
621 orc::SymbolAliasMap AliasMap;
622 for (auto &KV : *Flags)
623 if (!Allow || Allow(KV.first))
624 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
625
626 if (AliasMap.empty())
627 return Error::success();
628
629 // Define the re-exports.
630 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
631}
632
633LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
634 : IPLS(std::move(IPLS)) {}
635
636void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
637
638LookupState::LookupState() = default;
639LookupState::LookupState(LookupState &&) = default;
640LookupState &LookupState::operator=(LookupState &&) = default;
641LookupState::~LookupState() = default;
642
643void LookupState::continueLookup(Error Err) {
644 assert(IPLS && "Cannot call continueLookup on empty LookupState");
645 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
646 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
647}
648
649DefinitionGenerator::~DefinitionGenerator() {
650 std::deque<LookupState> LookupsToFail;
651 {
652 std::lock_guard<std::mutex> Lock(M);
653 std::swap(x&: PendingLookups, y&: LookupsToFail);
654 InUse = false;
655 }
656
657 for (auto &LS : LookupsToFail)
658 LS.continueLookup(Err: make_error<StringError>(
659 Args: "Query waiting on DefinitionGenerator that was destroyed",
660 Args: inconvertibleErrorCode()));
661}
662
663JITDylib::~JITDylib() {
664 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
665}
666
667Error JITDylib::clear() {
668 std::vector<ResourceTrackerSP> TrackersToRemove;
669 ES.runSessionLocked(F: [&]() {
670 assert(State != Closed && "JD is defunct");
671 for (auto &KV : TrackerSymbols)
672 TrackersToRemove.push_back(x: KV.first);
673 TrackersToRemove.push_back(x: getDefaultResourceTracker());
674 });
675
676 Error Err = Error::success();
677 for (auto &RT : TrackersToRemove)
678 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
679 return Err;
680}
681
682ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
683 return ES.runSessionLocked(F: [this] {
684 assert(State != Closed && "JD is defunct");
685 if (!DefaultTracker)
686 DefaultTracker = new ResourceTracker(this);
687 return DefaultTracker;
688 });
689}
690
691ResourceTrackerSP JITDylib::createResourceTracker() {
692 return ES.runSessionLocked(F: [this] {
693 assert(State == Open && "JD is defunct");
694 ResourceTrackerSP RT = new ResourceTracker(this);
695 return RT;
696 });
697}
698
699void JITDylib::removeGenerator(DefinitionGenerator &G) {
700 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
701 // session lock (since it may have to send errors to pending queries).
702 std::shared_ptr<DefinitionGenerator> TmpDG;
703
704 ES.runSessionLocked(F: [&] {
705 assert(State == Open && "JD is defunct");
706 auto I = llvm::find_if(Range&: DefGenerators,
707 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
708 return H.get() == &G;
709 });
710 assert(I != DefGenerators.end() && "Generator not found");
711 TmpDG = std::move(*I);
712 DefGenerators.erase(position: I);
713 });
714}
715
716Expected<SymbolFlagsMap>
717JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
718 SymbolFlagsMap SymbolFlags) {
719
720 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
721 if (FromMR.RT->isDefunct())
722 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
723
724 std::vector<NonOwningSymbolStringPtr> AddedSyms;
725 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
726
727 for (auto &[Name, Flags] : SymbolFlags) {
728 auto EntryItr = Symbols.find(Val: Name);
729
730 // If the entry already exists...
731 if (EntryItr != Symbols.end()) {
732
733 // If this is a strong definition then error out.
734 if (!Flags.isWeak()) {
735 // Remove any symbols already added.
736 for (auto &S : AddedSyms)
737 Symbols.erase(I: Symbols.find_as(Val: S));
738
739 // FIXME: Return all duplicates.
740 return make_error<DuplicateDefinition>(
741 Args: std::string(*Name), Args: "defineMaterializing operation");
742 }
743
744 // Otherwise just make a note to discard this symbol after the loop.
745 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
746 continue;
747 } else
748 EntryItr =
749 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
750
751 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
752 EntryItr->second.setState(SymbolState::Materializing);
753 }
754
755 // Remove any rejected weak definitions from the SymbolFlags map.
756 while (!RejectedWeakDefs.empty()) {
757 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
758 RejectedWeakDefs.pop_back();
759 }
760
761 return SymbolFlags;
762 });
763}
764
765Error JITDylib::replace(MaterializationResponsibility &FromMR,
766 std::unique_ptr<MaterializationUnit> MU) {
767 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
768 std::unique_ptr<MaterializationUnit> MustRunMU;
769 std::unique_ptr<MaterializationResponsibility> MustRunMR;
770
771 auto Err =
772 ES.runSessionLocked(F: [&, this]() -> Error {
773 if (FromMR.RT->isDefunct())
774 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
775
776#ifndef NDEBUG
777 for (auto &KV : MU->getSymbols()) {
778 auto SymI = Symbols.find(KV.first);
779 assert(SymI != Symbols.end() && "Replacing unknown symbol");
780 assert(SymI->second.getState() == SymbolState::Materializing &&
781 "Can not replace a symbol that ha is not materializing");
782 assert(!SymI->second.hasMaterializerAttached() &&
783 "Symbol should not have materializer attached already");
784 assert(UnmaterializedInfos.count(KV.first) == 0 &&
785 "Symbol being replaced should have no UnmaterializedInfo");
786 }
787#endif // NDEBUG
788
789 // If the tracker is defunct we need to bail out immediately.
790
791 // If any symbol has pending queries against it then we need to
792 // materialize MU immediately.
793 for (auto &KV : MU->getSymbols()) {
794 auto MII = MaterializingInfos.find(Val: KV.first);
795 if (MII != MaterializingInfos.end()) {
796 if (MII->second.hasQueriesPending()) {
797 MustRunMR = ES.createMaterializationResponsibility(
798 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
799 InitSymbol: std::move(MU->InitSymbol));
800 MustRunMU = std::move(MU);
801 return Error::success();
802 }
803 }
804 }
805
806 // Otherwise, make MU responsible for all the symbols.
807 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
808 args: FromMR.RT.get());
809 for (auto &KV : UMI->MU->getSymbols()) {
810 auto SymI = Symbols.find(Val: KV.first);
811 assert(SymI->second.getState() == SymbolState::Materializing &&
812 "Can not replace a symbol that is not materializing");
813 assert(!SymI->second.hasMaterializerAttached() &&
814 "Can not replace a symbol that has a materializer attached");
815 assert(UnmaterializedInfos.count(KV.first) == 0 &&
816 "Unexpected materializer entry in map");
817 SymI->second.setAddress(SymI->second.getAddress());
818 SymI->second.setMaterializerAttached(true);
819
820 auto &UMIEntry = UnmaterializedInfos[KV.first];
821 assert((!UMIEntry || !UMIEntry->MU) &&
822 "Replacing symbol with materializer still attached");
823 UMIEntry = UMI;
824 }
825
826 return Error::success();
827 });
828
829 if (Err)
830 return Err;
831
832 if (MustRunMU) {
833 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
834 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
835 args: std::move(MustRunMU), args: std::move(MustRunMR)));
836 } else {
837 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
838 }
839
840 return Error::success();
841}
842
843Expected<std::unique_ptr<MaterializationResponsibility>>
844JITDylib::delegate(MaterializationResponsibility &FromMR,
845 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
846
847 return ES.runSessionLocked(
848 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
849 if (FromMR.RT->isDefunct())
850 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
851
852 return ES.createMaterializationResponsibility(
853 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
854 });
855}
856
857SymbolNameSet
858JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
859 return ES.runSessionLocked(F: [&]() {
860 SymbolNameSet RequestedSymbols;
861
862 for (auto &KV : SymbolFlags) {
863 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
864 assert(Symbols.find(KV.first)->second.getState() !=
865 SymbolState::NeverSearched &&
866 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
867 "getRequestedSymbols can only be called for symbols that have "
868 "started materializing");
869 auto I = MaterializingInfos.find(Val: KV.first);
870 if (I == MaterializingInfos.end())
871 continue;
872
873 if (I->second.hasQueriesPending())
874 RequestedSymbols.insert(V: KV.first);
875 }
876
877 return RequestedSymbols;
878 });
879}
880
881Error JITDylib::resolve(MaterializationResponsibility &MR,
882 const SymbolMap &Resolved) {
883 AsynchronousSymbolQuerySet CompletedQueries;
884
885 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
886 if (MR.RT->isDefunct())
887 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
888
889 if (State != Open)
890 return make_error<StringError>(Args: "JITDylib " + getName() +
891 " is defunct",
892 Args: inconvertibleErrorCode());
893
894 struct WorklistEntry {
895 SymbolTable::iterator SymI;
896 ExecutorSymbolDef ResolvedSym;
897 };
898
899 SymbolNameSet SymbolsInErrorState;
900 std::vector<WorklistEntry> Worklist;
901 Worklist.reserve(n: Resolved.size());
902
903 // Build worklist and check for any symbols in the error state.
904 for (const auto &KV : Resolved) {
905
906 assert(!KV.second.getFlags().hasError() &&
907 "Resolution result can not have error flag set");
908
909 auto SymI = Symbols.find(Val: KV.first);
910
911 assert(SymI != Symbols.end() && "Symbol not found");
912 assert(!SymI->second.hasMaterializerAttached() &&
913 "Resolving symbol with materializer attached?");
914 assert(SymI->second.getState() == SymbolState::Materializing &&
915 "Symbol should be materializing");
916 assert(SymI->second.getAddress() == ExecutorAddr() &&
917 "Symbol has already been resolved");
918
919 if (SymI->second.getFlags().hasError())
920 SymbolsInErrorState.insert(V: KV.first);
921 else {
922 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
923 [[maybe_unused]] auto WeakOrCommon =
924 JITSymbolFlags::Weak | JITSymbolFlags::Common;
925 assert((KV.second.getFlags() & WeakOrCommon) &&
926 "Common symbols must be resolved as common or weak");
927 assert((KV.second.getFlags() & ~WeakOrCommon) ==
928 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
929 "Resolving symbol with incorrect flags");
930
931 } else
932 assert(KV.second.getFlags() == SymI->second.getFlags() &&
933 "Resolved flags should match the declared flags");
934
935 Worklist.push_back(
936 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
937 }
938 }
939
940 // If any symbols were in the error state then bail out.
941 if (!SymbolsInErrorState.empty()) {
942 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
943 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
944 return make_error<FailedToMaterialize>(
945 Args: getExecutionSession().getSymbolStringPool(),
946 Args: std::move(FailedSymbolsDepMap));
947 }
948
949 while (!Worklist.empty()) {
950 auto SymI = Worklist.back().SymI;
951 auto ResolvedSym = Worklist.back().ResolvedSym;
952 Worklist.pop_back();
953
954 auto &Name = SymI->first;
955
956 // Resolved symbols can not be weak: discard the weak flag.
957 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
958 SymI->second.setAddress(ResolvedSym.getAddress());
959 SymI->second.setFlags(ResolvedFlags);
960 SymI->second.setState(SymbolState::Resolved);
961
962 auto MII = MaterializingInfos.find(Val: Name);
963 if (MII == MaterializingInfos.end())
964 continue;
965
966 auto &MI = MII->second;
967 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
968 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
969 if (Q->isComplete())
970 CompletedQueries.insert(x: std::move(Q));
971 }
972 }
973
974 return Error::success();
975 }))
976 return Err;
977
978 // Otherwise notify all the completed queries.
979 for (auto &Q : CompletedQueries) {
980 assert(Q->isComplete() && "Q not completed");
981 Q->handleComplete(ES);
982 }
983
984 return Error::success();
985}
986
987void JITDylib::unlinkMaterializationResponsibility(
988 MaterializationResponsibility &MR) {
989 ES.runSessionLocked(F: [&]() {
990 auto I = TrackerMRs.find(Val: MR.RT.get());
991 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
992 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
993 I->second.erase(V: &MR);
994 if (I->second.empty())
995 TrackerMRs.erase(Val: MR.RT.get());
996 });
997}
998
999void JITDylib::shrinkMaterializationInfoMemory() {
1000 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1001 // memory since we may have long-lived JDs after linking is done.
1002
1003 if (UnmaterializedInfos.empty())
1004 UnmaterializedInfos.clear();
1005
1006 if (MaterializingInfos.empty())
1007 MaterializingInfos.clear();
1008}
1009
1010void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1011 bool LinkAgainstThisJITDylibFirst) {
1012 ES.runSessionLocked(F: [&]() {
1013 assert(State == Open && "JD is defunct");
1014 if (LinkAgainstThisJITDylibFirst) {
1015 LinkOrder.clear();
1016 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1017 LinkOrder.push_back(
1018 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1019 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1020 } else
1021 LinkOrder = std::move(NewLinkOrder);
1022 });
1023}
1024
1025void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1026 ES.runSessionLocked(F: [&]() {
1027 for (auto &KV : NewLinks) {
1028 // Skip elements of NewLinks that are already in the link order.
1029 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1030 continue;
1031
1032 LinkOrder.push_back(x: std::move(KV));
1033 }
1034 });
1035}
1036
1037void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1038 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1039}
1040
1041void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1042 JITDylibLookupFlags JDLookupFlags) {
1043 ES.runSessionLocked(F: [&]() {
1044 assert(State == Open && "JD is defunct");
1045 for (auto &KV : LinkOrder)
1046 if (KV.first == &OldJD) {
1047 KV = {&NewJD, JDLookupFlags};
1048 break;
1049 }
1050 });
1051}
1052
1053void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1054 ES.runSessionLocked(F: [&]() {
1055 assert(State == Open && "JD is defunct");
1056 auto I = llvm::find_if(Range&: LinkOrder,
1057 P: [&](const JITDylibSearchOrder::value_type &KV) {
1058 return KV.first == &JD;
1059 });
1060 if (I != LinkOrder.end())
1061 LinkOrder.erase(position: I);
1062 });
1063}
1064
1065Error JITDylib::remove(const SymbolNameSet &Names) {
1066 return ES.runSessionLocked(F: [&]() -> Error {
1067 assert(State == Open && "JD is defunct");
1068 SmallVector<SymbolStringPtr, 0> SymbolsToRemove;
1069 SymbolNameSet Missing;
1070 SymbolNameSet Materializing;
1071
1072 for (auto &Name : Names) {
1073 auto I = Symbols.find(Val: Name);
1074
1075 // Note symbol missing.
1076 if (I == Symbols.end()) {
1077 Missing.insert(V: Name);
1078 continue;
1079 }
1080
1081 // Note symbol materializing.
1082 if (I->second.getState() != SymbolState::NeverSearched &&
1083 I->second.getState() != SymbolState::Ready) {
1084 Materializing.insert(V: Name);
1085 continue;
1086 }
1087
1088 SymbolsToRemove.push_back(Elt: Name);
1089 }
1090
1091 // If any of the symbols are not defined, return an error.
1092 if (!Missing.empty())
1093 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1094 Args: std::move(Missing));
1095
1096 // If any of the symbols are currently materializing, return an error.
1097 if (!Materializing.empty())
1098 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1099 Args: std::move(Materializing));
1100
1101 // Remove the symbols. Erase by key rather than holding iterators across the
1102 // loop: a prior erase invalidates other stored iterators under
1103 // backward-shift deletion.
1104 for (const SymbolStringPtr &Name : SymbolsToRemove) {
1105 // If there is a materializer attached, call discard.
1106 auto UMII = UnmaterializedInfos.find(Val: Name);
1107 if (UMII != UnmaterializedInfos.end()) {
1108 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1109 UnmaterializedInfos.erase(I: UMII);
1110 }
1111
1112 Symbols.erase(Val: Name);
1113 }
1114
1115 shrinkMaterializationInfoMemory();
1116
1117 return Error::success();
1118 });
1119}
1120
1121void JITDylib::dump(raw_ostream &OS) {
1122 ES.runSessionLocked(F: [&, this]() {
1123 OS << "JITDylib \"" << getName() << "\" (ES: "
1124 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1125 << ", State = ";
1126 switch (State) {
1127 case Open:
1128 OS << "Open";
1129 break;
1130 case Closing:
1131 OS << "Closing";
1132 break;
1133 case Closed:
1134 OS << "Closed";
1135 break;
1136 }
1137 OS << ")\n";
1138 if (State == Closed)
1139 return;
1140 OS << "Link order: " << LinkOrder << "\n"
1141 << "Symbol table:\n";
1142
1143 // Sort symbols so we get a deterministic order and can check them in tests.
1144 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1145 for (auto &KV : Symbols)
1146 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1147 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1148 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1149
1150 for (auto &KV : SymbolsSorted) {
1151 OS << " \"" << *KV.first << "\": ";
1152 if (auto Addr = KV.second->getAddress())
1153 OS << Addr;
1154 else
1155 OS << "<not resolved> ";
1156
1157 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1158
1159 if (KV.second->hasMaterializerAttached()) {
1160 OS << " (Materializer ";
1161 auto I = UnmaterializedInfos.find(Val: KV.first);
1162 assert(I != UnmaterializedInfos.end() &&
1163 "Lazy symbol should have UnmaterializedInfo");
1164 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1165 } else
1166 OS << "\n";
1167 }
1168
1169 if (!MaterializingInfos.empty())
1170 OS << " MaterializingInfos entries:\n";
1171 for (auto &KV : MaterializingInfos) {
1172 OS << " \"" << *KV.first << "\":\n"
1173 << " " << KV.second.pendingQueries().size()
1174 << " pending queries: { ";
1175 for (const auto &Q : KV.second.pendingQueries())
1176 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1177 OS << "}\n";
1178 }
1179 });
1180}
1181
1182void JITDylib::MaterializingInfo::addQuery(
1183 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1184
1185 auto I = llvm::lower_bound(
1186 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1187 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1188 return V->getRequiredState() <= S;
1189 });
1190 PendingQueries.insert(position: I.base(), x: std::move(Q));
1191}
1192
1193void JITDylib::MaterializingInfo::removeQuery(
1194 const AsynchronousSymbolQuery &Q) {
1195 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1196 auto I = llvm::find_if(
1197 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1198 return V.get() == &Q;
1199 });
1200 if (I != PendingQueries.end())
1201 PendingQueries.erase(position: I);
1202}
1203
1204JITDylib::AsynchronousSymbolQueryList
1205JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1206 AsynchronousSymbolQueryList Result;
1207 while (!PendingQueries.empty()) {
1208 if (PendingQueries.back()->getRequiredState() > RequiredState)
1209 break;
1210
1211 Result.push_back(x: std::move(PendingQueries.back()));
1212 PendingQueries.pop_back();
1213 }
1214
1215 return Result;
1216}
1217
1218JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1219 : JITLinkDylib(std::move(Name)), ES(ES) {
1220 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1221}
1222
1223JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1224 // Note: Should be called under the session lock.
1225 assert(State != Closed && "JD is defunct");
1226
1227 SymbolNameVector SymbolsToRemove;
1228 SymbolNameVector SymbolsToFail;
1229
1230 if (&RT == DefaultTracker.get()) {
1231 SymbolNameSet TrackedSymbols;
1232 for (auto &KV : TrackerSymbols)
1233 TrackedSymbols.insert_range(R&: KV.second);
1234
1235 for (auto &KV : Symbols) {
1236 auto &Sym = KV.first;
1237 if (!TrackedSymbols.count(V: Sym))
1238 SymbolsToRemove.push_back(x: Sym);
1239 }
1240
1241 DefaultTracker.reset();
1242 } else {
1243 /// Check for a non-default tracker.
1244 auto I = TrackerSymbols.find(Val: &RT);
1245 if (I != TrackerSymbols.end()) {
1246 SymbolsToRemove = std::move(I->second);
1247 TrackerSymbols.erase(I);
1248 }
1249 // ... if not found this tracker was already defunct. Nothing to do.
1250 }
1251
1252 for (auto &Sym : SymbolsToRemove) {
1253 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1254
1255 // If there is a MaterializingInfo then collect any queries to fail.
1256 auto MII = MaterializingInfos.find(Val: Sym);
1257 if (MII != MaterializingInfos.end())
1258 SymbolsToFail.push_back(x: Sym);
1259 }
1260
1261 auto [QueriesToFail, FailedSymbols] =
1262 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1263
1264 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1265
1266 // Removed symbols should be taken out of the table altogether.
1267 for (auto &Sym : SymbolsToRemove) {
1268 auto I = Symbols.find(Val: Sym);
1269 assert(I != Symbols.end() && "Symbol not present in table");
1270
1271 // Remove Materializer if present.
1272 if (I->second.hasMaterializerAttached()) {
1273 // FIXME: Should this discard the symbols?
1274 auto J = UnmaterializedInfos.find(Val: Sym);
1275 assert(J != UnmaterializedInfos.end() &&
1276 "Symbol table indicates MU present, but no UMI record");
1277 if (J->second->MU)
1278 DefunctMUs.push_back(x: std::move(J->second->MU));
1279 UnmaterializedInfos.erase(I: J);
1280 } else {
1281 assert(!UnmaterializedInfos.count(Sym) &&
1282 "Symbol has materializer attached");
1283 }
1284
1285 Symbols.erase(I);
1286 }
1287
1288 shrinkMaterializationInfoMemory();
1289
1290 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1291 .DefunctMUs: std::move(DefunctMUs)};
1292}
1293
1294void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1295 assert(State != Closed && "JD is defunct");
1296 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1297 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1298 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1299
1300 // Update trackers for any not-yet materialized units.
1301 for (auto &KV : UnmaterializedInfos) {
1302 if (KV.second->RT == &SrcRT)
1303 KV.second->RT = &DstRT;
1304 }
1305
1306 // Update trackers for any active materialization responsibilities.
1307 {
1308 auto I = TrackerMRs.find(Val: &SrcRT);
1309 if (I != TrackerMRs.end()) {
1310 auto &SrcMRs = I->second;
1311 auto &DstMRs = TrackerMRs[&DstRT];
1312 for (auto *MR : SrcMRs)
1313 MR->RT = &DstRT;
1314 if (DstMRs.empty())
1315 DstMRs = std::move(SrcMRs);
1316 else
1317 DstMRs.insert_range(R&: SrcMRs);
1318 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1319 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1320 TrackerMRs.erase(Val: &SrcRT);
1321 }
1322 }
1323
1324 // If we're transfering to the default tracker we just need to delete the
1325 // tracked symbols for the source tracker.
1326 if (&DstRT == DefaultTracker.get()) {
1327 TrackerSymbols.erase(Val: &SrcRT);
1328 return;
1329 }
1330
1331 // If we're transferring from the default tracker we need to find all
1332 // currently untracked symbols.
1333 if (&SrcRT == DefaultTracker.get()) {
1334 assert(!TrackerSymbols.count(&SrcRT) &&
1335 "Default tracker should not appear in TrackerSymbols");
1336
1337 SymbolNameVector SymbolsToTrack;
1338
1339 SymbolNameSet CurrentlyTrackedSymbols;
1340 for (auto &KV : TrackerSymbols)
1341 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1342
1343 for (auto &KV : Symbols) {
1344 auto &Sym = KV.first;
1345 if (!CurrentlyTrackedSymbols.count(V: Sym))
1346 SymbolsToTrack.push_back(x: Sym);
1347 }
1348
1349 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1350 return;
1351 }
1352
1353 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1354
1355 // Finally if neither SrtRT or DstRT are the default tracker then
1356 // just append DstRT's tracked symbols to SrtRT's.
1357 auto SI = TrackerSymbols.find(Val: &SrcRT);
1358 if (SI == TrackerSymbols.end())
1359 return;
1360
1361 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1362 for (auto &Sym : SI->second)
1363 DstTrackedSymbols.push_back(x: std::move(Sym));
1364 TrackerSymbols.erase(I: SI);
1365}
1366
1367Error JITDylib::defineImpl(MaterializationUnit &MU) {
1368 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1369
1370 SymbolNameSet Duplicates;
1371 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1372 std::vector<SymbolStringPtr> MUDefsOverridden;
1373
1374 for (const auto &KV : MU.getSymbols()) {
1375 auto I = Symbols.find(Val: KV.first);
1376
1377 if (I != Symbols.end()) {
1378 if (KV.second.isStrong()) {
1379 if (I->second.getFlags().isStrong() ||
1380 I->second.getState() > SymbolState::NeverSearched)
1381 Duplicates.insert(V: KV.first);
1382 else {
1383 assert(I->second.getState() == SymbolState::NeverSearched &&
1384 "Overridden existing def should be in the never-searched "
1385 "state");
1386 ExistingDefsOverridden.push_back(x: KV.first);
1387 }
1388 } else
1389 MUDefsOverridden.push_back(x: KV.first);
1390 }
1391 }
1392
1393 // If there were any duplicate definitions then bail out.
1394 if (!Duplicates.empty()) {
1395 LLVM_DEBUG(
1396 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1397 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1398 Args: MU.getName().str());
1399 }
1400
1401 // Discard any overridden defs in this MU.
1402 LLVM_DEBUG({
1403 if (!MUDefsOverridden.empty())
1404 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1405 });
1406 for (auto &S : MUDefsOverridden)
1407 MU.doDiscard(JD: *this, Name: S);
1408
1409 // Discard existing overridden defs.
1410 LLVM_DEBUG({
1411 if (!ExistingDefsOverridden.empty())
1412 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1413 << "\n";
1414 });
1415 for (auto &S : ExistingDefsOverridden) {
1416
1417 auto UMII = UnmaterializedInfos.find(Val: S);
1418 assert(UMII != UnmaterializedInfos.end() &&
1419 "Overridden existing def should have an UnmaterializedInfo");
1420 UMII->second->MU->doDiscard(JD: *this, Name: S);
1421 }
1422
1423 // Finally, add the defs from this MU.
1424 for (auto &KV : MU.getSymbols()) {
1425 auto &SymEntry = Symbols[KV.first];
1426 SymEntry.setFlags(KV.second);
1427 SymEntry.setState(SymbolState::NeverSearched);
1428 SymEntry.setMaterializerAttached(true);
1429 }
1430
1431 return Error::success();
1432}
1433
1434void JITDylib::installMaterializationUnit(
1435 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1436
1437 /// defineImpl succeeded.
1438 if (&RT != DefaultTracker.get()) {
1439 auto &TS = TrackerSymbols[&RT];
1440 TS.reserve(n: TS.size() + MU->getSymbols().size());
1441 for (auto &KV : MU->getSymbols())
1442 TS.push_back(x: KV.first);
1443 }
1444
1445 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1446 for (auto &KV : UMI->MU->getSymbols())
1447 UnmaterializedInfos[KV.first] = UMI;
1448}
1449
1450void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1451 const SymbolNameSet &QuerySymbols) {
1452 for (auto &QuerySymbol : QuerySymbols) {
1453 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1454 if (MII != MaterializingInfos.end())
1455 MII->second.removeQuery(Q);
1456 }
1457}
1458
1459Platform::~Platform() = default;
1460
1461Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1462 ExecutionSession &ES,
1463 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1464
1465 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1466 Error CompoundErr = Error::success();
1467 std::mutex LookupMutex;
1468 std::condition_variable CV;
1469 uint64_t Count = InitSyms.size();
1470
1471 LLVM_DEBUG({
1472 dbgs() << "Issuing init-symbol lookup:\n";
1473 for (auto &KV : InitSyms)
1474 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1475 });
1476
1477 for (auto &KV : InitSyms) {
1478 auto *JD = KV.first;
1479 auto Names = std::move(KV.second);
1480 ES.lookup(
1481 K: LookupKind::Static,
1482 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1483 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1484 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1485 {
1486 std::lock_guard<std::mutex> Lock(LookupMutex);
1487 --Count;
1488 if (Result) {
1489 assert(!CompoundResult.count(JD) &&
1490 "Duplicate JITDylib in lookup?");
1491 CompoundResult[JD] = std::move(*Result);
1492 } else
1493 CompoundErr =
1494 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1495 }
1496 CV.notify_one();
1497 },
1498 RegisterDependencies: NoDependenciesToRegister);
1499 }
1500
1501 std::unique_lock<std::mutex> Lock(LookupMutex);
1502 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1503
1504 if (CompoundErr)
1505 return std::move(CompoundErr);
1506
1507 return std::move(CompoundResult);
1508}
1509
1510void Platform::lookupInitSymbolsAsync(
1511 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1512 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1513
1514 class TriggerOnComplete {
1515 public:
1516 using OnCompleteFn = unique_function<void(Error)>;
1517 TriggerOnComplete(OnCompleteFn OnComplete)
1518 : OnComplete(std::move(OnComplete)) {}
1519 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1520 void reportResult(Error Err) {
1521 std::lock_guard<std::mutex> Lock(ResultMutex);
1522 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1523 }
1524
1525 private:
1526 std::mutex ResultMutex;
1527 Error LookupResult{Error::success()};
1528 OnCompleteFn OnComplete;
1529 };
1530
1531 LLVM_DEBUG({
1532 dbgs() << "Issuing init-symbol lookup:\n";
1533 for (auto &KV : InitSyms)
1534 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1535 });
1536
1537 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1538
1539 for (auto &KV : InitSyms) {
1540 auto *JD = KV.first;
1541 auto Names = std::move(KV.second);
1542 ES.lookup(
1543 K: LookupKind::Static,
1544 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1545 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1546 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1547 TOC->reportResult(Err: Result.takeError());
1548 },
1549 RegisterDependencies: NoDependenciesToRegister);
1550 }
1551}
1552
1553MaterializationTask::~MaterializationTask() {
1554 // If this task wasn't run then fail materialization.
1555 if (MR)
1556 MR->failMaterialization();
1557}
1558
1559void MaterializationTask::printDescription(raw_ostream &OS) {
1560 OS << "Materialization task: " << MU->getName() << " in "
1561 << MR->getTargetJITDylib().getName();
1562}
1563
1564void MaterializationTask::run() {
1565 assert(MU && "MU should not be null");
1566 assert(MR && "MR should not be null");
1567 MU->materialize(R: std::move(MR));
1568}
1569
1570void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1571
1572void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1573
1574ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1575 : EPC(std::move(EPC)), BootstrapJD(createBareJITDylib(Name: "<bootstrap>")) {
1576 // Associated EPC and this.
1577 this->EPC->ES = this;
1578 SymbolMap BootstrapSymbols;
1579 for (auto &[Name, Ptr] : this->EPC->getBootstrapSymbolsMap())
1580 BootstrapSymbols[intern(SymName: Name)] =
1581 ExecutorSymbolDef(Ptr, JITSymbolFlags::Exported);
1582 // Can't fail: BootstrapJD is a new, empty JD and the BootstrapSymbols
1583 // variable is a map, so can't contain duplicates.
1584 cantFail(Err: BootstrapJD.define(MU: absoluteSymbols(Symbols: std::move(BootstrapSymbols))));
1585}
1586
1587ExecutionSession::~ExecutionSession() {
1588 // You must call endSession prior to destroying the session.
1589 assert(!SessionOpen &&
1590 "Session still open. Did you forget to call endSession?");
1591}
1592
1593Error ExecutionSession::endSession() {
1594 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1595
1596 WaitingOnGraph::OpRecorder *GOpRecorderToEnd = nullptr;
1597 auto JDsToRemove = runSessionLocked(F: [&] {
1598
1599#ifdef EXPENSIVE_CHECKS
1600 verifySessionState("Entering ExecutionSession::endSession");
1601#endif
1602
1603 if (SessionOpen)
1604 GOpRecorderToEnd = GOpRecorder;
1605 SessionOpen = false;
1606 return JDs;
1607 });
1608
1609 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1610
1611 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1612
1613 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1614
1615 if (GOpRecorderToEnd)
1616 GOpRecorderToEnd->recordEnd();
1617
1618 return Err;
1619}
1620
1621void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1622 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1623}
1624
1625void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1626 runSessionLocked(F: [&] {
1627 assert(!ResourceManagers.empty() && "No managers registered");
1628 if (ResourceManagers.back() == &RM)
1629 ResourceManagers.pop_back();
1630 else {
1631 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1632 assert(I != ResourceManagers.end() && "RM not registered");
1633 ResourceManagers.erase(position: I);
1634 }
1635 });
1636}
1637
1638JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1639 return runSessionLocked(F: [&, this]() -> JITDylib * {
1640 for (auto &JD : JDs)
1641 if (JD->getName() == Name)
1642 return JD.get();
1643 return nullptr;
1644 });
1645}
1646
1647JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1648 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1649 return runSessionLocked(F: [&, this]() -> JITDylib & {
1650 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1651 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1652 return *JDs.back();
1653 });
1654}
1655
1656Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1657 auto &JD = createBareJITDylib(Name);
1658 if (P)
1659 if (auto Err = P->setupJITDylib(JD))
1660 return std::move(Err);
1661 return JD;
1662}
1663
1664Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1665 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1666 runSessionLocked(F: [&] {
1667 for (auto &JD : JDsToRemove) {
1668 assert(JD->State == JITDylib::Open && "JD already closed");
1669 JD->State = JITDylib::Closing;
1670 auto I = llvm::find(Range&: JDs, Val: JD);
1671 assert(I != JDs.end() && "JD does not appear in session JDs");
1672 JDs.erase(position: I);
1673 }
1674 });
1675
1676 // Clear JITDylibs and notify the platform.
1677 Error Err = Error::success();
1678 for (auto JD : JDsToRemove) {
1679 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1680 if (P)
1681 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1682 }
1683
1684 // Set JD to closed state. Clear remaining data structures.
1685 runSessionLocked(F: [&] {
1686 for (auto &JD : JDsToRemove) {
1687 assert(JD->State == JITDylib::Closing && "JD should be closing");
1688 JD->State = JITDylib::Closed;
1689 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1690 assert(JD->UnmaterializedInfos.empty() &&
1691 "JD.UnmaterializedInfos is not empty after clear");
1692 assert(JD->MaterializingInfos.empty() &&
1693 "JD.MaterializingInfos is not empty after clear");
1694 assert(JD->TrackerSymbols.empty() &&
1695 "TrackerSymbols is not empty after clear");
1696 JD->DefGenerators.clear();
1697 JD->LinkOrder.clear();
1698 }
1699 });
1700
1701 return Err;
1702}
1703
1704Expected<std::vector<JITDylibSP>>
1705JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1706 if (JDs.empty())
1707 return std::vector<JITDylibSP>();
1708
1709 auto &ES = JDs.front()->getExecutionSession();
1710 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1711 DenseSet<JITDylib *> Visited;
1712 std::vector<JITDylibSP> Result;
1713
1714 for (auto &JD : JDs) {
1715
1716 if (JD->State != Open)
1717 return make_error<StringError>(
1718 Args: "Error building link order: " + JD->getName() + " is defunct",
1719 Args: inconvertibleErrorCode());
1720 if (Visited.count(V: JD.get()))
1721 continue;
1722
1723 SmallVector<JITDylibSP, 64> WorkStack;
1724 WorkStack.push_back(Elt: JD);
1725 Visited.insert(V: JD.get());
1726
1727 while (!WorkStack.empty()) {
1728 Result.push_back(x: std::move(WorkStack.back()));
1729 WorkStack.pop_back();
1730
1731 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1732 auto &JD = *KV.first;
1733 if (!Visited.insert(V: &JD).second)
1734 continue;
1735 WorkStack.push_back(Elt: &JD);
1736 }
1737 }
1738 }
1739 return Result;
1740 });
1741}
1742
1743Expected<std::vector<JITDylibSP>>
1744JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1745 auto Result = getDFSLinkOrder(JDs);
1746 if (Result)
1747 std::reverse(first: Result->begin(), last: Result->end());
1748 return Result;
1749}
1750
1751Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1752 return getDFSLinkOrder(JDs: {this});
1753}
1754
1755Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1756 return getReverseDFSLinkOrder(JDs: {this});
1757}
1758
1759void ExecutionSession::lookupFlags(
1760 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1761 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1762
1763 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1764 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1765 args: std::move(OnComplete)),
1766 Err: Error::success());
1767}
1768
1769Expected<SymbolFlagsMap>
1770ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1771 SymbolLookupSet LookupSet) {
1772
1773 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1774 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1775 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1776 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1777 ResultP.set_value(std::move(Result));
1778 }),
1779 Err: Error::success());
1780
1781 auto ResultF = ResultP.get_future();
1782 return ResultF.get();
1783}
1784
1785void ExecutionSession::lookup(
1786 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1787 SymbolLookupSet Symbols, SymbolState RequiredState,
1788 SymbolsResolvedCallback NotifyComplete,
1789 RegisterDependenciesFunction RegisterDependencies) {
1790
1791 LLVM_DEBUG({
1792 runSessionLocked([&]() {
1793 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1794 << " (required state: " << RequiredState << ")\n";
1795 });
1796 });
1797
1798 // lookup can be re-entered recursively if running on a single thread. Run any
1799 // outstanding MUs in case this query depends on them, otherwise this lookup
1800 // will starve waiting for a result from an MU that is stuck in the queue.
1801 dispatchOutstandingMUs();
1802
1803 auto Unresolved = std::move(Symbols);
1804 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1805 args: std::move(NotifyComplete));
1806
1807 auto IPLS = std::make_unique<InProgressFullLookupState>(
1808 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1809 args: std::move(RegisterDependencies));
1810
1811 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1812}
1813
1814Expected<SymbolMap>
1815ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1816 SymbolLookupSet Symbols, LookupKind K,
1817 SymbolState RequiredState,
1818 RegisterDependenciesFunction RegisterDependencies) {
1819#if LLVM_ENABLE_THREADS
1820 // In the threaded case we use promises to return the results.
1821 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1822
1823 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1824 PromisedResult.set_value(std::move(R));
1825 };
1826
1827#else
1828 SymbolMap Result;
1829 Error ResolutionError = Error::success();
1830
1831 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1832 ErrorAsOutParameter _(ResolutionError);
1833 if (R)
1834 Result = std::move(*R);
1835 else
1836 ResolutionError = R.takeError();
1837 };
1838#endif
1839
1840 // Perform the asynchronous lookup.
1841 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1842 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1843
1844#if LLVM_ENABLE_THREADS
1845 return PromisedResult.get_future().get();
1846#else
1847 if (ResolutionError)
1848 return std::move(ResolutionError);
1849
1850 return Result;
1851#endif
1852}
1853
1854Expected<ExecutorSymbolDef>
1855ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1856 SymbolStringPtr Name, SymbolState RequiredState) {
1857 SymbolLookupSet Names({Name});
1858
1859 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1860 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1861 assert(ResultMap->size() == 1 && "Unexpected number of results");
1862 assert(ResultMap->count(Name) && "Missing result for symbol");
1863 return std::move(ResultMap->begin()->second);
1864 } else
1865 return ResultMap.takeError();
1866}
1867
1868Expected<ExecutorSymbolDef>
1869ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1870 SymbolState RequiredState) {
1871 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1872}
1873
1874Expected<ExecutorSymbolDef>
1875ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1876 SymbolState RequiredState) {
1877 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1878}
1879
1880Error ExecutionSession::registerJITDispatchHandlers(
1881 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1882
1883 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1884 Symbols: SymbolLookupSet::fromMapKeys(
1885 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1886 if (!TagSyms)
1887 return TagSyms.takeError();
1888
1889 // Associate tag addresses with implementations.
1890 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1891
1892 // Check that no tags are being overwritten.
1893 for (auto &[TagName, TagSym] : *TagSyms) {
1894 auto TagAddr = TagSym.getAddress();
1895 if (JITDispatchHandlers.count(Val: TagAddr))
1896 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1897 " (for " + *TagName +
1898 ") already registered",
1899 Args: inconvertibleErrorCode());
1900 }
1901
1902 // At this point we're guaranteed to succeed. Install the handlers.
1903 for (auto &[TagName, TagSym] : *TagSyms) {
1904 auto TagAddr = TagSym.getAddress();
1905 auto I = WFs.find(Val: TagName);
1906 assert(I != WFs.end() && I->second &&
1907 "JITDispatchHandler implementation missing");
1908 JITDispatchHandlers[TagAddr] =
1909 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1910 LLVM_DEBUG({
1911 dbgs() << "Associated function tag \"" << *TagName << "\" ("
1912 << formatv("{0:x}", TagAddr) << ") with handler\n";
1913 });
1914 }
1915
1916 return Error::success();
1917}
1918
1919void ExecutionSession::runJITDispatchHandler(
1920 SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr,
1921 shared::WrapperFunctionBuffer ArgBytes) {
1922
1923 std::shared_ptr<JITDispatchHandlerFunction> F;
1924 {
1925 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1926 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1927 if (I != JITDispatchHandlers.end())
1928 F = I->second;
1929 }
1930
1931 if (F)
1932 (*F)(std::move(SendResult), ArgBytes.data(), ArgBytes.size());
1933 else
1934 SendResult(shared::WrapperFunctionBuffer::createOutOfBandError(
1935 Msg: ("No function registered for tag " +
1936 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1937 .str()));
1938}
1939
1940void ExecutionSession::dump(raw_ostream &OS) {
1941 runSessionLocked(F: [this, &OS]() {
1942 for (auto &JD : JDs)
1943 JD->dump(OS);
1944 });
1945}
1946
1947#ifdef EXPENSIVE_CHECKS
1948bool ExecutionSession::verifySessionState(Twine Phase) {
1949 return runSessionLocked([&]() {
1950 bool AllOk = true;
1951
1952 for (auto &JD : JDs) {
1953
1954 auto LogFailure = [&]() -> raw_fd_ostream & {
1955 auto &Stream = errs();
1956 if (AllOk)
1957 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1958 << "\n";
1959 Stream << " In JITDylib " << JD->getName() << ", ";
1960 AllOk = false;
1961 return Stream;
1962 };
1963
1964 if (JD->State != JITDylib::Open) {
1965 LogFailure()
1966 << "state is not Open, but JD is in ExecutionSession list.";
1967 }
1968
1969 // Check symbol table.
1970 // 1. If the entry state isn't resolved then check that no address has
1971 // been set.
1972 // 2. Check that if the hasMaterializerAttached flag is set then there is
1973 // an UnmaterializedInfo entry, and vice-versa.
1974 for (auto &[Sym, Entry] : JD->Symbols) {
1975 // Check that unresolved symbols have null addresses.
1976 if (Entry.getState() < SymbolState::Resolved) {
1977 if (Entry.getAddress()) {
1978 LogFailure() << "symbol " << Sym << " has state "
1979 << Entry.getState()
1980 << " (not-yet-resolved) but non-null address "
1981 << Entry.getAddress() << ".\n";
1982 }
1983 }
1984
1985 // Check that the hasMaterializerAttached flag is correct.
1986 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
1987 if (Entry.hasMaterializerAttached()) {
1988 if (UMIItr == JD->UnmaterializedInfos.end()) {
1989 LogFailure() << "symbol " << Sym
1990 << " entry claims materializer attached, but "
1991 "UnmaterializedInfos has no corresponding entry.\n";
1992 }
1993 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
1994 LogFailure()
1995 << "symbol " << Sym
1996 << " entry claims no materializer attached, but "
1997 "UnmaterializedInfos has an unexpected entry for it.\n";
1998 }
1999 }
2000
2001 // Check that every UnmaterializedInfo entry has a corresponding entry
2002 // in the Symbols table.
2003 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2004 auto SymItr = JD->Symbols.find(Sym);
2005 if (SymItr == JD->Symbols.end()) {
2006 LogFailure()
2007 << "symbol " << Sym
2008 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2009 }
2010 }
2011
2012 // Check consistency of the MaterializingInfos table.
2013 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2014
2015 auto SymItr = JD->Symbols.find(Sym);
2016 if (SymItr == JD->Symbols.end()) {
2017 // If there's no Symbols entry for this MaterializingInfos entry then
2018 // report that.
2019 LogFailure()
2020 << "symbol " << Sym
2021 << " has MaterializingInfos entry, but no Symbols entry.\n";
2022 } else {
2023 // Otherwise check consistency between Symbols and MaterializingInfos.
2024
2025 // Ready symbols should not have MaterializingInfos.
2026 if (SymItr->second.getState() == SymbolState::Ready) {
2027 LogFailure()
2028 << "symbol " << Sym
2029 << " is in Ready state, should not have MaterializingInfo.\n";
2030 }
2031
2032 // Pending queries should be for subsequent states.
2033 auto CurState = static_cast<SymbolState>(
2034 static_cast<std::underlying_type_t<SymbolState>>(
2035 SymItr->second.getState()) + 1);
2036 for (auto &Q : MII.PendingQueries) {
2037 if (Q->getRequiredState() != CurState) {
2038 if (Q->getRequiredState() > CurState)
2039 CurState = Q->getRequiredState();
2040 else
2041 LogFailure() << "symbol " << Sym
2042 << " has stale or misordered queries.\n";
2043 }
2044 }
2045 }
2046 }
2047 }
2048
2049 return AllOk;
2050 });
2051}
2052#endif // EXPENSIVE_CHECKS
2053
2054void ExecutionSession::dispatchOutstandingMUs() {
2055 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2056 while (true) {
2057 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2058 std::unique_ptr<MaterializationResponsibility>>>
2059 JMU;
2060
2061 {
2062 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2063 if (!OutstandingMUs.empty()) {
2064 JMU.emplace(args: std::move(OutstandingMUs.back()));
2065 OutstandingMUs.pop_back();
2066 }
2067 }
2068
2069 if (!JMU)
2070 break;
2071
2072 assert(JMU->first && "No MU?");
2073 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2074 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2075 args: std::move(JMU->second)));
2076 }
2077 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2078}
2079
2080Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2081 LLVM_DEBUG({
2082 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2083 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2084 });
2085 std::vector<ResourceManager *> CurrentResourceManagers;
2086
2087 JITDylib::RemoveTrackerResult R;
2088
2089 runSessionLocked(F: [&] {
2090 CurrentResourceManagers = ResourceManagers;
2091 RT.makeDefunct();
2092 R = RT.getJITDylib().IL_removeTracker(RT);
2093 });
2094
2095 // Release any defunct MaterializationUnits.
2096 R.DefunctMUs.clear();
2097
2098 Error Err = Error::success();
2099
2100 auto &JD = RT.getJITDylib();
2101 for (auto *L : reverse(C&: CurrentResourceManagers))
2102 Err = joinErrors(E1: std::move(Err),
2103 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2104
2105 for (auto &Q : R.QueriesToFail)
2106 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2107 Args&: R.FailedSymbols));
2108
2109 return Err;
2110}
2111
2112void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2113 ResourceTracker &SrcRT) {
2114 LLVM_DEBUG({
2115 dbgs() << "In " << SrcRT.getJITDylib().getName()
2116 << " transfering resources from tracker "
2117 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2118 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2119 });
2120
2121 // No-op transfers are allowed and do not invalidate the source.
2122 if (&DstRT == &SrcRT)
2123 return;
2124
2125 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2126 "Can't transfer resources between JITDylibs");
2127 runSessionLocked(F: [&]() {
2128 SrcRT.makeDefunct();
2129 auto &JD = DstRT.getJITDylib();
2130 JD.transferTracker(DstRT, SrcRT);
2131 for (auto *L : reverse(C&: ResourceManagers))
2132 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2133 SrcK: SrcRT.getKeyUnsafe());
2134 });
2135}
2136
2137void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2138 runSessionLocked(F: [&]() {
2139 LLVM_DEBUG({
2140 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2141 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2142 });
2143 if (!RT.isDefunct())
2144 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2145 SrcRT&: RT);
2146 });
2147}
2148
2149Error ExecutionSession::IL_updateCandidatesFor(
2150 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2151 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2152 return Candidates.forEachWithRemoval(
2153 Body: [&](const SymbolStringPtr &Name,
2154 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2155 /// Search for the symbol. If not found then continue without
2156 /// removal.
2157 auto SymI = JD.Symbols.find(Val: Name);
2158 if (SymI == JD.Symbols.end())
2159 return false;
2160
2161 // If this is a non-exported symbol and we're matching exported
2162 // symbols only then remove this symbol from the candidates list.
2163 //
2164 // If we're tracking non-candidates then add this to the non-candidate
2165 // list.
2166 if (!SymI->second.getFlags().isExported() &&
2167 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2168 if (NonCandidates)
2169 NonCandidates->add(Name, Flags: SymLookupFlags);
2170 return true;
2171 }
2172
2173 // If we match against a materialization-side-effects only symbol
2174 // then make sure it is weakly-referenced. Otherwise bail out with
2175 // an error.
2176 // FIXME: Use a "materialization-side-effects-only symbols must be
2177 // weakly referenced" specific error here to reduce confusion.
2178 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2179 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2180 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2181 Args: SymbolNameVector({Name}));
2182
2183 // If we matched against this symbol but it is in the error state
2184 // then bail out and treat it as a failure to materialize.
2185 if (SymI->second.getFlags().hasError()) {
2186 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2187 (*FailedSymbolsMap)[&JD] = {Name};
2188 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2189 Args: std::move(FailedSymbolsMap));
2190 }
2191
2192 // Otherwise this is a match. Remove it from the candidate set.
2193 return true;
2194 });
2195}
2196
2197void ExecutionSession::OL_resumeLookupAfterGeneration(
2198 InProgressLookupState &IPLS) {
2199
2200 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2201 "Should not be called for not-in-generator lookups");
2202 IPLS.GenState = InProgressLookupState::NotInGenerator;
2203
2204 LookupState LS;
2205
2206 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2207 IPLS.CurDefGeneratorStack.pop_back();
2208 std::lock_guard<std::mutex> Lock(DG->M);
2209
2210 // If there are no pending lookups then mark the generator as free and
2211 // return.
2212 if (DG->PendingLookups.empty()) {
2213 DG->InUse = false;
2214 return;
2215 }
2216
2217 // Otherwise resume the next lookup.
2218 LS = std::move(DG->PendingLookups.front());
2219 DG->PendingLookups.pop_front();
2220 }
2221
2222 if (LS.IPLS) {
2223 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2224 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2225 }
2226}
2227
2228void ExecutionSession::OL_applyQueryPhase1(
2229 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2230
2231 LLVM_DEBUG({
2232 dbgs() << "Entering OL_applyQueryPhase1:\n"
2233 << " Lookup kind: " << IPLS->K << "\n"
2234 << " Search order: " << IPLS->SearchOrder
2235 << ", Current index = " << IPLS->CurSearchOrderIndex
2236 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2237 << " Lookup set: " << IPLS->LookupSet << "\n"
2238 << " Definition generator candidates: "
2239 << IPLS->DefGeneratorCandidates << "\n"
2240 << " Definition generator non-candidates: "
2241 << IPLS->DefGeneratorNonCandidates << "\n";
2242 });
2243
2244 if (IPLS->GenState == InProgressLookupState::InGenerator)
2245 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2246
2247 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2248 "Lookup should not be in InGenerator state here");
2249
2250 // FIXME: We should attach the query as we go: This provides a result in a
2251 // single pass in the common case where all symbols have already reached the
2252 // required state. The query could be detached again in the 'fail' method on
2253 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2254
2255 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2256
2257 // If we've been handed an error or received one back from a generator then
2258 // fail the query. We don't need to unlink: At this stage the query hasn't
2259 // actually been lodged.
2260 if (Err)
2261 return IPLS->fail(Err: std::move(Err));
2262
2263 // Get the next JITDylib and lookup flags.
2264 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2265 auto &JD = *KV.first;
2266 auto JDLookupFlags = KV.second;
2267
2268 LLVM_DEBUG({
2269 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2270 << ") with lookup set " << IPLS->LookupSet << ":\n";
2271 });
2272
2273 // If we've just reached a new JITDylib then perform some setup.
2274 if (IPLS->NewJITDylib) {
2275 // Add any non-candidates from the last JITDylib (if any) back on to the
2276 // list of definition candidates for this JITDylib, reset definition
2277 // non-candidates to the empty set.
2278 SymbolLookupSet Tmp;
2279 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2280 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2281
2282 LLVM_DEBUG({
2283 dbgs() << " First time visiting " << JD.getName()
2284 << ", resetting candidate sets and building generator stack\n";
2285 });
2286
2287 // Build the definition generator stack for this JITDylib.
2288 runSessionLocked(F: [&] {
2289 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2290 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2291 R: reverse(C&: JD.DefGenerators));
2292 });
2293
2294 // Flag that we've done our initialization.
2295 IPLS->NewJITDylib = false;
2296 }
2297
2298 // Remove any generation candidates that are already defined (and match) in
2299 // this JITDylib.
2300 runSessionLocked(F: [&] {
2301 // Update the list of candidates (and non-candidates) for definition
2302 // generation.
2303 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2304 Err = IL_updateCandidatesFor(
2305 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2306 NonCandidates: JD.DefGenerators.empty() ? nullptr
2307 : &IPLS->DefGeneratorNonCandidates);
2308 LLVM_DEBUG({
2309 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2310 << "\n";
2311 });
2312
2313 // If this lookup was resumed after auto-suspension but all candidates
2314 // have already been generated (by some previous call to the generator)
2315 // treat the lookup as if it had completed generation.
2316 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2317 IPLS->DefGeneratorCandidates.empty())
2318 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2319 });
2320
2321 // If we encountered an error while filtering generation candidates then
2322 // bail out.
2323 if (Err)
2324 return IPLS->fail(Err: std::move(Err));
2325
2326 /// Apply any definition generators on the stack.
2327 LLVM_DEBUG({
2328 if (IPLS->CurDefGeneratorStack.empty())
2329 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2330 else if (IPLS->DefGeneratorCandidates.empty())
2331 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2332 else
2333 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2334 << " remaining generators for "
2335 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2336 });
2337 while (!IPLS->CurDefGeneratorStack.empty() &&
2338 !IPLS->DefGeneratorCandidates.empty()) {
2339 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2340
2341 if (!DG)
2342 return IPLS->fail(Err: make_error<StringError>(
2343 Args: "DefinitionGenerator removed while lookup in progress",
2344 Args: inconvertibleErrorCode()));
2345
2346 // At this point the lookup is in either the NotInGenerator state, or in
2347 // the ResumedForGenerator state.
2348 // If this lookup is in the NotInGenerator state then check whether the
2349 // generator is in use. If the generator is not in use then move the
2350 // lookup to the InGenerator state and continue. If the generator is
2351 // already in use then just add this lookup to the pending lookups list
2352 // and bail out.
2353 // If this lookup is in the ResumedForGenerator state then just move it
2354 // to InGenerator and continue.
2355 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2356 std::lock_guard<std::mutex> Lock(DG->M);
2357 if (DG->InUse) {
2358 DG->PendingLookups.push_back(x: std::move(IPLS));
2359 return;
2360 }
2361 DG->InUse = true;
2362 }
2363
2364 IPLS->GenState = InProgressLookupState::InGenerator;
2365
2366 auto K = IPLS->K;
2367 auto &LookupSet = IPLS->DefGeneratorCandidates;
2368
2369 // Run the generator. If the generator takes ownership of QA then this
2370 // will break the loop.
2371 {
2372 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2373 LookupState LS(std::move(IPLS));
2374 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2375 IPLS = std::move(LS.IPLS);
2376 }
2377
2378 // If the lookup returned then pop the generator stack and unblock the
2379 // next lookup on this generator (if any).
2380 if (IPLS)
2381 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2382
2383 // If there was an error then fail the query.
2384 if (Err) {
2385 LLVM_DEBUG({
2386 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2387 });
2388 assert(IPLS && "LS cannot be retained if error is returned");
2389 return IPLS->fail(Err: std::move(Err));
2390 }
2391
2392 // Otherwise if QA was captured then break the loop.
2393 if (!IPLS) {
2394 LLVM_DEBUG(
2395 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2396 return;
2397 }
2398
2399 // Otherwise if we're continuing around the loop then update candidates
2400 // for the next round.
2401 runSessionLocked(F: [&] {
2402 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2403 Err = IL_updateCandidatesFor(
2404 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2405 NonCandidates: JD.DefGenerators.empty() ? nullptr
2406 : &IPLS->DefGeneratorNonCandidates);
2407 });
2408
2409 // If updating candidates failed then fail the query.
2410 if (Err) {
2411 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2412 return IPLS->fail(Err: std::move(Err));
2413 }
2414 }
2415
2416 if (IPLS->DefGeneratorCandidates.empty() &&
2417 IPLS->DefGeneratorNonCandidates.empty()) {
2418 // Early out if there are no remaining symbols.
2419 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2420 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2421 break;
2422 } else {
2423 // If we get here then we've moved on to the next JITDylib with candidates
2424 // remaining.
2425 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2426 ++IPLS->CurSearchOrderIndex;
2427 IPLS->NewJITDylib = true;
2428 }
2429 }
2430
2431 // Remove any weakly referenced candidates that could not be found/generated.
2432 IPLS->DefGeneratorCandidates.remove_if(
2433 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2434 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2435 });
2436
2437 // If we get here then we've finished searching all JITDylibs.
2438 // If we matched all symbols then move to phase 2, otherwise fail the query
2439 // with a SymbolsNotFound error.
2440 if (IPLS->DefGeneratorCandidates.empty()) {
2441 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2442 IPLS->complete(IPLS: std::move(IPLS));
2443 } else {
2444 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2445 IPLS->fail(Err: make_error<SymbolsNotFound>(
2446 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2447 }
2448}
2449
2450void ExecutionSession::OL_completeLookup(
2451 std::unique_ptr<InProgressLookupState> IPLS,
2452 std::shared_ptr<AsynchronousSymbolQuery> Q,
2453 RegisterDependenciesFunction RegisterDependencies) {
2454
2455 LLVM_DEBUG({
2456 dbgs() << "Entering OL_completeLookup:\n"
2457 << " Lookup kind: " << IPLS->K << "\n"
2458 << " Search order: " << IPLS->SearchOrder
2459 << ", Current index = " << IPLS->CurSearchOrderIndex
2460 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2461 << " Lookup set: " << IPLS->LookupSet << "\n"
2462 << " Definition generator candidates: "
2463 << IPLS->DefGeneratorCandidates << "\n"
2464 << " Definition generator non-candidates: "
2465 << IPLS->DefGeneratorNonCandidates << "\n";
2466 });
2467
2468 bool QueryComplete = false;
2469 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2470
2471 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2472 for (auto &KV : IPLS->SearchOrder) {
2473 auto &JD = *KV.first;
2474 auto JDLookupFlags = KV.second;
2475 LLVM_DEBUG({
2476 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2477 << ") with lookup set " << IPLS->LookupSet << ":\n";
2478 });
2479
2480 auto Err = IPLS->LookupSet.forEachWithRemoval(
2481 Body: [&](const SymbolStringPtr &Name,
2482 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2483 LLVM_DEBUG({
2484 dbgs() << " Attempting to match \"" << Name << "\" ("
2485 << SymLookupFlags << ")... ";
2486 });
2487
2488 /// Search for the symbol. If not found then continue without
2489 /// removal.
2490 auto SymI = JD.Symbols.find(Val: Name);
2491 if (SymI == JD.Symbols.end()) {
2492 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2493 return false;
2494 }
2495
2496 // If this is a non-exported symbol and we're matching exported
2497 // symbols only then skip this symbol without removal.
2498 if (!SymI->second.getFlags().isExported() &&
2499 JDLookupFlags ==
2500 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2501 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2502 return false;
2503 }
2504
2505 // If we match against a materialization-side-effects only symbol
2506 // then make sure it is weakly-referenced. Otherwise bail out with
2507 // an error.
2508 // FIXME: Use a "materialization-side-effects-only symbols must be
2509 // weakly referenced" specific error here to reduce confusion.
2510 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2511 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2512 LLVM_DEBUG({
2513 dbgs() << "error: "
2514 "required, but symbol is has-side-effects-only\n";
2515 });
2516 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2517 Args: SymbolNameVector({Name}));
2518 }
2519
2520 // If we matched against this symbol but it is in the error state
2521 // then bail out and treat it as a failure to materialize.
2522 if (SymI->second.getFlags().hasError()) {
2523 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2524 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2525 (*FailedSymbolsMap)[&JD] = {Name};
2526 return make_error<FailedToMaterialize>(
2527 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2528 }
2529
2530 // Otherwise this is a match.
2531
2532 // If this symbol is already in the required state then notify the
2533 // query, remove the symbol and continue.
2534 if (SymI->second.getState() >= Q->getRequiredState()) {
2535 LLVM_DEBUG(dbgs()
2536 << "matched, symbol already in required state\n");
2537 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2538
2539 // If this symbol is in anything other than the Ready state then
2540 // we need to track the dependence.
2541 if (SymI->second.getState() != SymbolState::Ready)
2542 Q->addQueryDependence(JD, Name);
2543
2544 return true;
2545 }
2546
2547 // Otherwise this symbol does not yet meet the required state. Check
2548 // whether it has a materializer attached, and if so prepare to run
2549 // it.
2550 if (SymI->second.hasMaterializerAttached()) {
2551 assert(SymI->second.getAddress() == ExecutorAddr() &&
2552 "Symbol not resolved but already has address?");
2553 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2554 assert(UMII != JD.UnmaterializedInfos.end() &&
2555 "Lazy symbol should have UnmaterializedInfo");
2556
2557 auto UMI = UMII->second;
2558 assert(UMI->MU && "Materializer should not be null");
2559 assert(UMI->RT && "Tracker should not be null");
2560 LLVM_DEBUG({
2561 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2562 << " (" << UMI->MU->getName() << ")\n";
2563 });
2564
2565 // Move all symbols associated with this MaterializationUnit into
2566 // materializing state.
2567 for (auto &KV : UMI->MU->getSymbols()) {
2568 auto SymK = JD.Symbols.find(Val: KV.first);
2569 assert(SymK != JD.Symbols.end() &&
2570 "No entry for symbol covered by MaterializationUnit");
2571 SymK->second.setMaterializerAttached(false);
2572 SymK->second.setState(SymbolState::Materializing);
2573 JD.UnmaterializedInfos.erase(Val: KV.first);
2574 }
2575
2576 // Add MU to the list of MaterializationUnits to be materialized.
2577 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2578 } else
2579 LLVM_DEBUG(dbgs() << "matched, registering query");
2580
2581 // Add the query to the PendingQueries list and continue, deleting
2582 // the element from the lookup set.
2583 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2584 SymI->second.getState() != SymbolState::Ready &&
2585 "By this line the symbol should be materializing");
2586 auto &MI = JD.MaterializingInfos[Name];
2587 MI.addQuery(Q);
2588 Q->addQueryDependence(JD, Name);
2589
2590 return true;
2591 });
2592
2593 JD.shrinkMaterializationInfoMemory();
2594
2595 // Handle failure.
2596 if (Err) {
2597
2598 LLVM_DEBUG({
2599 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2600 });
2601
2602 // Detach the query.
2603 Q->detach();
2604
2605 // Replace the MUs.
2606 for (auto &KV : CollectedUMIs) {
2607 auto &JD = *KV.first;
2608 for (auto &UMI : KV.second)
2609 for (auto &KV2 : UMI->MU->getSymbols()) {
2610 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2611 "Unexpected materializer in map");
2612 auto SymI = JD.Symbols.find(Val: KV2.first);
2613 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2614 assert(SymI->second.getState() == SymbolState::Materializing &&
2615 "Can not replace symbol that is not materializing");
2616 assert(!SymI->second.hasMaterializerAttached() &&
2617 "MaterializerAttached flag should not be set");
2618 SymI->second.setMaterializerAttached(true);
2619 JD.UnmaterializedInfos[KV2.first] = UMI;
2620 }
2621 }
2622
2623 return Err;
2624 }
2625 }
2626
2627 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2628 IPLS->LookupSet.forEachWithRemoval(
2629 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2630 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2631 Q->dropSymbol(Name);
2632 return true;
2633 } else
2634 return false;
2635 });
2636
2637 if (!IPLS->LookupSet.empty()) {
2638 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2639 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2640 Args: IPLS->LookupSet.getSymbolNames());
2641 }
2642
2643 // Record whether the query completed.
2644 QueryComplete = Q->isComplete();
2645
2646 LLVM_DEBUG({
2647 dbgs() << "Query successfully "
2648 << (QueryComplete ? "completed" : "lodged") << "\n";
2649 });
2650
2651 // Move the collected MUs to the OutstandingMUs list.
2652 if (!CollectedUMIs.empty()) {
2653 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2654
2655 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2656 for (auto &KV : CollectedUMIs) {
2657 LLVM_DEBUG({
2658 auto &JD = *KV.first;
2659 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2660 << " MUs.\n";
2661 });
2662 for (auto &UMI : KV.second) {
2663 auto MR = createMaterializationResponsibility(
2664 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2665 InitSymbol: std::move(UMI->MU->InitSymbol));
2666 OutstandingMUs.push_back(
2667 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2668 }
2669 }
2670 } else
2671 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2672
2673 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2674 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2675 RegisterDependencies(Q->QueryRegistrations);
2676 } else
2677 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2678
2679 return Error::success();
2680 });
2681
2682 if (LodgingErr) {
2683 LLVM_DEBUG(dbgs() << "Failing query\n");
2684 Q->detach();
2685 Q->handleFailed(Err: std::move(LodgingErr));
2686 return;
2687 }
2688
2689 if (QueryComplete) {
2690 LLVM_DEBUG(dbgs() << "Completing query\n");
2691 Q->handleComplete(ES&: *this);
2692 }
2693
2694 dispatchOutstandingMUs();
2695}
2696
2697void ExecutionSession::OL_completeLookupFlags(
2698 std::unique_ptr<InProgressLookupState> IPLS,
2699 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2700
2701 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2702 LLVM_DEBUG({
2703 dbgs() << "Entering OL_completeLookupFlags:\n"
2704 << " Lookup kind: " << IPLS->K << "\n"
2705 << " Search order: " << IPLS->SearchOrder
2706 << ", Current index = " << IPLS->CurSearchOrderIndex
2707 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2708 << " Lookup set: " << IPLS->LookupSet << "\n"
2709 << " Definition generator candidates: "
2710 << IPLS->DefGeneratorCandidates << "\n"
2711 << " Definition generator non-candidates: "
2712 << IPLS->DefGeneratorNonCandidates << "\n";
2713 });
2714
2715 SymbolFlagsMap Result;
2716
2717 // Attempt to find flags for each symbol.
2718 for (auto &KV : IPLS->SearchOrder) {
2719 auto &JD = *KV.first;
2720 auto JDLookupFlags = KV.second;
2721 LLVM_DEBUG({
2722 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2723 << ") with lookup set " << IPLS->LookupSet << ":\n";
2724 });
2725
2726 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2727 SymbolLookupFlags SymLookupFlags) {
2728 LLVM_DEBUG({
2729 dbgs() << " Attempting to match \"" << Name << "\" ("
2730 << SymLookupFlags << ")... ";
2731 });
2732
2733 // Search for the symbol. If not found then continue without removing
2734 // from the lookup set.
2735 auto SymI = JD.Symbols.find(Val: Name);
2736 if (SymI == JD.Symbols.end()) {
2737 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2738 return false;
2739 }
2740
2741 // If this is a non-exported symbol then it doesn't match. Skip it.
2742 if (!SymI->second.getFlags().isExported() &&
2743 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2744 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2745 return false;
2746 }
2747
2748 LLVM_DEBUG({
2749 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2750 << "\n";
2751 });
2752 Result[Name] = SymI->second.getFlags();
2753 return true;
2754 });
2755 }
2756
2757 // Remove any weakly referenced symbols that haven't been resolved.
2758 IPLS->LookupSet.remove_if(
2759 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2760 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2761 });
2762
2763 if (!IPLS->LookupSet.empty()) {
2764 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2765 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2766 Args: IPLS->LookupSet.getSymbolNames());
2767 }
2768
2769 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2770 return Result;
2771 });
2772
2773 // Run the callback on the result.
2774 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2775 OnComplete(std::move(Result));
2776}
2777
2778void ExecutionSession::OL_destroyMaterializationResponsibility(
2779 MaterializationResponsibility &MR) {
2780
2781 assert(MR.SymbolFlags.empty() &&
2782 "All symbols should have been explicitly materialized or failed");
2783 MR.JD.unlinkMaterializationResponsibility(MR);
2784}
2785
2786SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2787 const MaterializationResponsibility &MR) {
2788 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2789}
2790
2791Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2792 const SymbolMap &Symbols) {
2793 LLVM_DEBUG({
2794 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2795 });
2796#ifndef NDEBUG
2797 for (auto &KV : Symbols) {
2798 auto I = MR.SymbolFlags.find(KV.first);
2799 assert(I != MR.SymbolFlags.end() &&
2800 "Resolving symbol outside this responsibility set");
2801 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2802 "Can't resolve materialization-side-effects-only symbol");
2803 if (I->second & JITSymbolFlags::Common) {
2804 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2805 assert((KV.second.getFlags() & WeakOrCommon) &&
2806 "Common symbols must be resolved as common or weak");
2807 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2808 (I->second & ~JITSymbolFlags::Common) &&
2809 "Resolving symbol with incorrect flags");
2810 } else
2811 assert(KV.second.getFlags() == I->second &&
2812 "Resolving symbol with incorrect flags");
2813 }
2814#endif
2815
2816 return MR.JD.resolve(MR, Resolved: Symbols);
2817}
2818
2819WaitingOnGraph::ExternalState
2820ExecutionSession::IL_getSymbolState(JITDylib *JD,
2821 NonOwningSymbolStringPtr Name) {
2822 if (JD->State != JITDylib::Open)
2823 return WaitingOnGraph::ExternalState::Failed;
2824
2825 auto I = JD->Symbols.find_as(Val: Name);
2826
2827 // FIXME: Can we eliminate this possibility if we support query binding?
2828 if (I == JD->Symbols.end())
2829 return WaitingOnGraph::ExternalState::Failed;
2830
2831 if (I->second.getFlags().hasError())
2832 return WaitingOnGraph::ExternalState::Failed;
2833
2834 if (I->second.getState() == SymbolState::Ready)
2835 return WaitingOnGraph::ExternalState::Ready;
2836
2837 return WaitingOnGraph::ExternalState::None;
2838}
2839
2840template <typename UpdateSymbolFn, typename UpdateQueryFn>
2841void ExecutionSession::IL_collectQueries(
2842 JITDylib::AsynchronousSymbolQuerySet &Qs,
2843 WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
2844 UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
2845
2846 for (auto &[JD, Symbols] : QualifiedSymbols) {
2847 // IL_emit and JITDylib removal are synchronized by the session lock.
2848 // Since JITDylib removal removes any contained nodes from the
2849 // WaitingOnGraph, we should be able to assert that all nodes in the
2850 // WaitingOnGraph have not been removed.
2851 assert(JD->State == JITDylib::Open &&
2852 "WaitingOnGraph includes definition in defunct JITDylib");
2853 for (auto &Symbol : Symbols) {
2854 // Update symbol table.
2855 auto I = JD->Symbols.find_as(Val: Symbol);
2856 assert(I != JD->Symbols.end() &&
2857 "Failed Symbol missing from JD symbol table");
2858 auto &Entry = I->second;
2859 UpdateSymbol(Entry);
2860
2861 // Collect queries.
2862 auto J = JD->MaterializingInfos.find_as(Val: Symbol);
2863 if (J != JD->MaterializingInfos.end()) {
2864 for (auto &Q : J->second.takeAllPendingQueries()) {
2865 UpdateQuery(*Q, *JD, Symbol, Entry);
2866 Qs.insert(x: std::move(Q));
2867 }
2868 JD->MaterializingInfos.erase(I: J);
2869 }
2870 }
2871 }
2872}
2873
2874Expected<ExecutionSession::EmitQueries>
2875ExecutionSession::IL_emit(MaterializationResponsibility &MR,
2876 WaitingOnGraph::SimplifyResult SR) {
2877
2878 if (MR.RT->isDefunct())
2879 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
2880
2881 auto &TargetJD = MR.getTargetJITDylib();
2882 if (TargetJD.State != JITDylib::Open)
2883 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
2884 " is defunct",
2885 Args: inconvertibleErrorCode());
2886
2887#ifdef EXPENSIVE_CHECKS
2888 verifySessionState("entering ExecutionSession::IL_emit");
2889#endif
2890
2891 auto ER = G.emit(SR: std::move(SR),
2892 GetExternalState: [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
2893 return IL_getSymbolState(JD, Name);
2894 });
2895
2896 EmitQueries EQ;
2897
2898 // Handle failed queries.
2899 for (auto &SN : ER.Failed)
2900 IL_collectQueries(
2901 Qs&: EQ.Failed, QualifiedSymbols&: SN->defs(),
2902 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) {
2903 E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
2904 },
2905 UpdateQuery: [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
2906 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2907 auto &FS = EQ.FailedSymsForQuery[&Q];
2908 if (!FS)
2909 FS = std::make_shared<SymbolDependenceMap>();
2910 (*FS)[&JD].insert(V: SymbolStringPtr(Name));
2911 });
2912
2913 for (auto &FQ : EQ.Failed)
2914 FQ->detach();
2915
2916 for (auto &SN : ER.Ready)
2917 IL_collectQueries(
2918 Qs&: EQ.Completed, QualifiedSymbols&: SN->defs(),
2919 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
2920 UpdateQuery: [](AsynchronousSymbolQuery &Q, JITDylib &JD,
2921 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2922 Q.notifySymbolMetRequiredState(Name: SymbolStringPtr(Name), Sym: E.getSymbol());
2923 });
2924
2925 // std::erase_if is not available in C++17, and llvm::erase_if does not work
2926 // here.
2927 for (auto it = EQ.Completed.begin(), end = EQ.Completed.end(); it != end;) {
2928 if ((*it)->isComplete()) {
2929 ++it;
2930 } else {
2931 it = EQ.Completed.erase(position: it);
2932 }
2933 }
2934
2935#ifdef EXPENSIVE_CHECKS
2936 verifySessionState("exiting ExecutionSession::IL_emit");
2937#endif
2938
2939 return std::move(EQ);
2940}
2941
2942Error ExecutionSession::OL_notifyEmitted(
2943 MaterializationResponsibility &MR,
2944 ArrayRef<SymbolDependenceGroup> DepGroups) {
2945 LLVM_DEBUG({
2946 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
2947 << "\n";
2948 if (!DepGroups.empty()) {
2949 dbgs() << " Initial dependencies:\n";
2950 for (auto &SDG : DepGroups) {
2951 dbgs() << " Symbols: " << SDG.Symbols
2952 << ", Dependencies: " << SDG.Dependencies << "\n";
2953 }
2954 }
2955 });
2956
2957#ifndef NDEBUG
2958 SymbolNameSet Visited;
2959 for (auto &DG : DepGroups) {
2960 for (auto &Sym : DG.Symbols) {
2961 assert(MR.SymbolFlags.count(Sym) &&
2962 "DG contains dependence for symbol outside this MR");
2963 assert(Visited.insert(Sym).second &&
2964 "DG contains duplicate entries for Name");
2965 }
2966 }
2967#endif // NDEBUG
2968
2969 std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
2970 WaitingOnGraph::ContainerElementsMap Residual;
2971 {
2972 auto &JDResidual = Residual[&MR.getTargetJITDylib()];
2973 for (auto &[Name, Flags] : MR.getSymbols())
2974 JDResidual.insert(V: NonOwningSymbolStringPtr(Name));
2975
2976 for (auto &SDG : DepGroups) {
2977 WaitingOnGraph::ContainerElementsMap Defs;
2978 assert(!SDG.Symbols.empty());
2979 auto &JDDefs = Defs[&MR.getTargetJITDylib()];
2980 for (auto &Def : SDG.Symbols) {
2981 JDDefs.insert(V: NonOwningSymbolStringPtr(Def));
2982 JDResidual.erase(V: NonOwningSymbolStringPtr(Def));
2983 }
2984 WaitingOnGraph::ContainerElementsMap Deps;
2985 if (!SDG.Dependencies.empty()) {
2986 for (auto &[JD, Syms] : SDG.Dependencies) {
2987 auto &JDDeps = Deps[JD];
2988 for (auto &Dep : Syms)
2989 JDDeps.insert(V: NonOwningSymbolStringPtr(Dep));
2990 }
2991 }
2992 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2993 args: std::move(Defs), args: std::move(Deps)));
2994 }
2995 if (!JDResidual.empty())
2996 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2997 args: std::move(Residual), args: WaitingOnGraph::ContainerElementsMap()));
2998 }
2999
3000 auto SR = WaitingOnGraph::simplify(SNs: std::move(SNs), Rec: GOpRecorder);
3001
3002 LLVM_DEBUG({
3003 dbgs() << " Simplified dependencies:\n";
3004 for (auto &SN : SR.superNodes()) {
3005
3006 auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
3007 std::vector<JITDylib *> JDs;
3008 for (auto &[JD, _] : C)
3009 JDs.push_back(JD);
3010 llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
3011 return LHS->getName() < RHS->getName();
3012 });
3013 return JDs;
3014 };
3015
3016 auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
3017 std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
3018 llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
3019 const NonOwningSymbolStringPtr &RHS) {
3020 return *LHS < *RHS;
3021 });
3022 return Names;
3023 };
3024
3025 dbgs() << " Defs: {";
3026 for (auto *JD : SortedLibs(SN->defs())) {
3027 dbgs() << " (" << JD->getName() << ", [";
3028 for (auto &Sym : SortedNames(SN->defs()[JD]))
3029 dbgs() << " " << Sym;
3030 dbgs() << " ])";
3031 }
3032 dbgs() << " }, Deps: {";
3033 for (auto *JD : SortedLibs(SN->deps())) {
3034 dbgs() << " (" << JD->getName() << ", [";
3035 for (auto &Sym : SortedNames(SN->deps()[JD]))
3036 dbgs() << " " << Sym;
3037 dbgs() << " ])";
3038 }
3039 dbgs() << " }\n";
3040 }
3041 });
3042 auto EmitQueries =
3043 runSessionLocked(F: [&]() { return IL_emit(MR, SR: std::move(SR)); });
3044
3045 // On error bail out.
3046 if (!EmitQueries)
3047 return EmitQueries.takeError();
3048
3049 // Otherwise notify failed queries, and any updated queries that have been
3050 // completed.
3051
3052 // FIXME: Get rid of error return from notifyEmitted.
3053 SymbolDependenceMap BadDeps;
3054 {
3055 for (auto &FQ : EmitQueries->Failed) {
3056 FQ->detach();
3057 assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
3058 "Missing failed symbols for query");
3059 auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
3060 for (auto &[JD, Syms] : *FailedSyms) {
3061 auto &BadDepsForJD = BadDeps[JD];
3062 for (auto &Sym : Syms)
3063 BadDepsForJD.insert(V: Sym);
3064 }
3065 FQ->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
3066 Args: std::move(FailedSyms)));
3067 }
3068 }
3069
3070 for (auto &UQ : EmitQueries->Completed)
3071 UQ->handleComplete(ES&: *this);
3072
3073 // If there are any bad dependencies then return an error.
3074 if (!BadDeps.empty()) {
3075 SymbolNameSet BadNames;
3076 // Note: The name set calculated here is bogus: it includes all symbols in
3077 // the MR, not just the ones that failed. We want to remove the error
3078 // return path from notifyEmitted anyway, so this is just a brief
3079 // placeholder to maintain (roughly) the current error behavior.
3080 for (auto &[Name, Flags] : MR.getSymbols())
3081 BadNames.insert(V: Name);
3082 MR.SymbolFlags.clear();
3083 return make_error<UnsatisfiedSymbolDependencies>(
3084 Args: getSymbolStringPool(), Args: &MR.getTargetJITDylib(), Args: std::move(BadNames),
3085 Args: std::move(BadDeps), Args: "dependencies removed or in error state");
3086 }
3087
3088 MR.SymbolFlags.clear();
3089 return Error::success();
3090}
3091
3092Error ExecutionSession::OL_defineMaterializing(
3093 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3094
3095 LLVM_DEBUG({
3096 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3097 << NewSymbolFlags << "\n";
3098 });
3099 if (auto AcceptedDefs =
3100 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3101 // Add all newly accepted symbols to this responsibility object.
3102 for (auto &KV : *AcceptedDefs)
3103 MR.SymbolFlags.insert(KV);
3104 return Error::success();
3105 } else
3106 return AcceptedDefs.takeError();
3107}
3108
3109std::pair<JITDylib::AsynchronousSymbolQuerySet,
3110 std::shared_ptr<SymbolDependenceMap>>
3111ExecutionSession::IL_failSymbols(JITDylib &JD,
3112 const SymbolNameVector &SymbolsToFail) {
3113
3114#ifdef EXPENSIVE_CHECKS
3115 verifySessionState("entering ExecutionSession::IL_failSymbols");
3116#endif
3117
3118 // Early out in the easy case.
3119 if (SymbolsToFail.empty())
3120 return {};
3121
3122 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3123 auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
3124 auto I = FailJD->Symbols.find_as(Val: FailSym);
3125 assert(I != FailJD->Symbols.end());
3126 I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
3127 auto J = FailJD->MaterializingInfos.find_as(Val: FailSym);
3128 if (J != FailJD->MaterializingInfos.end()) {
3129 for (auto &Q : J->second.takeAllPendingQueries())
3130 FailedQueries.insert(x: std::move(Q));
3131 FailJD->MaterializingInfos.erase(I: J);
3132 }
3133 };
3134
3135 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3136
3137 {
3138 auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
3139 for (auto &Sym : SymbolsToFail) {
3140 FailedSymsForJD.insert(V: Sym);
3141 Fail(&JD, NonOwningSymbolStringPtr(Sym));
3142 }
3143 }
3144
3145 WaitingOnGraph::ContainerElementsMap ToFail;
3146 auto &JDToFail = ToFail[&JD];
3147 for (auto &Sym : SymbolsToFail)
3148 JDToFail.insert(V: NonOwningSymbolStringPtr(Sym));
3149
3150 auto FailedSNs = G.fail(Failed: ToFail, Rec: GOpRecorder);
3151
3152 for (auto &SN : FailedSNs) {
3153 for (auto &[FailJD, Defs] : SN->defs()) {
3154 auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
3155 for (auto &Def : Defs) {
3156 FailedSymsForFailJD.insert(V: SymbolStringPtr(Def));
3157 Fail(FailJD, Def);
3158 }
3159 }
3160 }
3161
3162 // Detach all failed queries.
3163 for (auto &Q : FailedQueries)
3164 Q->detach();
3165
3166#ifdef EXPENSIVE_CHECKS
3167 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3168#endif
3169
3170 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3171}
3172
3173void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3174
3175 LLVM_DEBUG({
3176 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3177 << MR.SymbolFlags << "\n";
3178 });
3179
3180 if (MR.SymbolFlags.empty())
3181 return;
3182
3183 SymbolNameVector SymbolsToFail;
3184 for (auto &[Name, Flags] : MR.SymbolFlags)
3185 SymbolsToFail.push_back(x: Name);
3186 MR.SymbolFlags.clear();
3187
3188 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3189 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3190
3191 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3192 // If the tracker is defunct then there's nothing to do here.
3193 if (MR.RT->isDefunct())
3194 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3195 std::shared_ptr<SymbolDependenceMap>>();
3196 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3197 });
3198
3199 for (auto &Q : FailedQueries) {
3200 Q->detach();
3201 Q->handleFailed(
3202 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3203 }
3204}
3205
3206Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3207 std::unique_ptr<MaterializationUnit> MU) {
3208 for (auto &KV : MU->getSymbols()) {
3209 assert(MR.SymbolFlags.count(KV.first) &&
3210 "Replacing definition outside this responsibility set");
3211 MR.SymbolFlags.erase(Val: KV.first);
3212 }
3213
3214 if (MU->getInitializerSymbol() == MR.InitSymbol)
3215 MR.InitSymbol = nullptr;
3216
3217 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3218 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3219 << "\n";
3220 }););
3221
3222 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3223}
3224
3225Expected<std::unique_ptr<MaterializationResponsibility>>
3226ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3227 const SymbolNameSet &Symbols) {
3228
3229 SymbolStringPtr DelegatedInitSymbol;
3230 SymbolFlagsMap DelegatedFlags;
3231
3232 for (auto &Name : Symbols) {
3233 auto I = MR.SymbolFlags.find(Val: Name);
3234 assert(I != MR.SymbolFlags.end() &&
3235 "Symbol is not tracked by this MaterializationResponsibility "
3236 "instance");
3237
3238 DelegatedFlags[Name] = std::move(I->second);
3239 if (Name == MR.InitSymbol)
3240 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3241
3242 MR.SymbolFlags.erase(I);
3243 }
3244
3245 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3246 InitSymbol: std::move(DelegatedInitSymbol));
3247}
3248
3249#ifndef NDEBUG
3250void ExecutionSession::dumpDispatchInfo(Task &T) {
3251 runSessionLocked([&]() {
3252 dbgs() << "Dispatching: ";
3253 T.printDescription(dbgs());
3254 dbgs() << "\n";
3255 });
3256}
3257#endif // NDEBUG
3258
3259} // End namespace orc.
3260} // End namespace llvm.
3261