1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
14#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
16#include "llvm/Support/FormatVariadic.h"
17#include "llvm/Support/MSVCErrorWorkarounds.h"
18#include "llvm/Support/raw_ostream.h"
19
20#include <condition_variable>
21#include <future>
22#include <optional>
23
24#define DEBUG_TYPE "orc"
25
26namespace llvm {
27namespace orc {
28
29char ResourceTrackerDefunct::ID = 0;
30char JITDylibDefunct::ID = 0;
31char FailedToMaterialize::ID = 0;
32char SymbolsNotFound::ID = 0;
33char SymbolsCouldNotBeRemoved::ID = 0;
34char MissingSymbolDefinitions::ID = 0;
35char UnexpectedSymbolDefinitions::ID = 0;
36char UnsatisfiedSymbolDependencies::ID = 0;
37char MaterializationTask::ID = 0;
38char LookupTask::ID = 0;
39
40RegisterDependenciesFunction NoDependenciesToRegister =
41 RegisterDependenciesFunction();
42
43void MaterializationUnit::anchor() {}
44
45ResourceTracker::ResourceTracker(JITDylibSP JD) {
46 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
47 "JITDylib must be two byte aligned");
48 JD->Retain();
49 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
50}
51
52ResourceTracker::~ResourceTracker() {
53 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
54 getJITDylib().Release();
55}
56
57Error ResourceTracker::remove() {
58 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
59}
60
61void ResourceTracker::transferTo(ResourceTracker &DstRT) {
62 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
63}
64
65void ResourceTracker::makeDefunct() {
66 uintptr_t Val = JDAndFlag.load();
67 Val |= 0x1U;
68 JDAndFlag.store(i: Val);
69}
70
71ResourceManager::~ResourceManager() = default;
72
73ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
74 : RT(std::move(RT)) {}
75
76std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
77 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
78}
79
80void ResourceTrackerDefunct::log(raw_ostream &OS) const {
81 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
82}
83
84std::error_code JITDylibDefunct::convertToErrorCode() const {
85 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
86}
87
88void JITDylibDefunct::log(raw_ostream &OS) const {
89 OS << "JITDylib " << JD->getName() << " (" << (void *)JD.get()
90 << ") is defunct";
91}
92
93FailedToMaterialize::FailedToMaterialize(
94 std::shared_ptr<SymbolStringPool> SSP,
95 std::shared_ptr<SymbolDependenceMap> Symbols)
96 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
97 assert(this->SSP && "String pool cannot be null");
98 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
99
100 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
101 // don't have to manually retain/release.
102 for (auto &[JD, Syms] : *this->Symbols)
103 JD->Retain();
104}
105
106FailedToMaterialize::~FailedToMaterialize() {
107 for (auto &[JD, Syms] : *Symbols)
108 JD->Release();
109}
110
111std::error_code FailedToMaterialize::convertToErrorCode() const {
112 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
113}
114
115void FailedToMaterialize::log(raw_ostream &OS) const {
116 OS << "Failed to materialize symbols: " << *Symbols;
117}
118
119UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
120 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
121 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
122 std::string Explanation)
123 : SSP(std::move(SSP)), JD(std::move(JD)),
124 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
125 Explanation(std::move(Explanation)) {}
126
127std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
128 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
129}
130
131void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
132 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
133 << ", due to unsatisfied dependencies " << BadDeps;
134 if (!Explanation.empty())
135 OS << " (" << Explanation << ")";
136}
137
138SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
139 SymbolNameSet Symbols)
140 : SSP(std::move(SSP)) {
141 llvm::append_range(C&: this->Symbols, R&: Symbols);
142 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
143}
144
145SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
146 SymbolNameVector Symbols)
147 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
148 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
149}
150
151std::error_code SymbolsNotFound::convertToErrorCode() const {
152 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
153}
154
155void SymbolsNotFound::log(raw_ostream &OS) const {
156 OS << "Symbols not found: " << Symbols;
157}
158
159SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
160 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
161 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
162 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
163}
164
165std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
166 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
167}
168
169void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
170 OS << "Symbols could not be removed: " << Symbols;
171}
172
173std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
174 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
175}
176
177void MissingSymbolDefinitions::log(raw_ostream &OS) const {
178 OS << "Missing definitions in module " << ModuleName
179 << ": " << Symbols;
180}
181
182std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
183 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
184}
185
186void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
187 OS << "Unexpected definitions in module " << ModuleName
188 << ": " << Symbols;
189}
190
191void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
192 JD->getExecutionSession().lookup(
193 K: LookupKind::Static, SearchOrder: {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
194 Symbols: SymbolLookupSet(Name), RequiredState: SymbolState::Ready,
195 NotifyComplete: [OnComplete = std::move(OnComplete)
196#ifndef NDEBUG
197 ,
198 Name = this->Name // Captured for the assert below only.
199#endif // NDEBUG
200 ](Expected<SymbolMap> Result) mutable {
201 if (Result) {
202 assert(Result->size() == 1 && "Unexpected number of results");
203 assert(Result->count(Name) &&
204 "Result does not contain expected symbol");
205 OnComplete(Result->begin()->second);
206 } else
207 OnComplete(Result.takeError());
208 },
209 RegisterDependencies: NoDependenciesToRegister);
210}
211
212AsynchronousSymbolQuery::AsynchronousSymbolQuery(
213 const SymbolLookupSet &Symbols, SymbolState RequiredState,
214 SymbolsResolvedCallback NotifyComplete)
215 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
216 assert(RequiredState >= SymbolState::Resolved &&
217 "Cannot query for a symbols that have not reached the resolve state "
218 "yet");
219
220 OutstandingSymbolsCount = Symbols.size();
221
222 for (auto &[Name, Flags] : Symbols)
223 ResolvedSymbols[Name] = ExecutorSymbolDef();
224}
225
226void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
227 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
228 auto I = ResolvedSymbols.find(Val: Name);
229 assert(I != ResolvedSymbols.end() &&
230 "Resolving symbol outside the requested set");
231 assert(I->second == ExecutorSymbolDef() &&
232 "Redundantly resolving symbol Name");
233
234 // If this is a materialization-side-effects-only symbol then drop it,
235 // otherwise update its map entry with its resolved address.
236 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
237 ResolvedSymbols.erase(I);
238 else
239 I->second = std::move(Sym);
240 --OutstandingSymbolsCount;
241}
242
243void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
244 assert(OutstandingSymbolsCount == 0 &&
245 "Symbols remain, handleComplete called prematurely");
246
247 class RunQueryCompleteTask : public Task {
248 public:
249 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
250 SymbolsResolvedCallback NotifyComplete)
251 : ResolvedSymbols(std::move(ResolvedSymbols)),
252 NotifyComplete(std::move(NotifyComplete)) {}
253 void printDescription(raw_ostream &OS) override {
254 OS << "Execute query complete callback for " << ResolvedSymbols;
255 }
256 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
257
258 private:
259 SymbolMap ResolvedSymbols;
260 SymbolsResolvedCallback NotifyComplete;
261 };
262
263 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
264 args: std::move(NotifyComplete));
265 NotifyComplete = SymbolsResolvedCallback();
266 ES.dispatchTask(T: std::move(T));
267}
268
269void AsynchronousSymbolQuery::handleFailed(Error Err) {
270 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
271 OutstandingSymbolsCount == 0 &&
272 "Query should already have been abandoned");
273 NotifyComplete(std::move(Err));
274 NotifyComplete = SymbolsResolvedCallback();
275}
276
277void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
278 SymbolStringPtr Name) {
279 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
280 (void)Added;
281 assert(Added && "Duplicate dependence notification?");
282}
283
284void AsynchronousSymbolQuery::removeQueryDependence(
285 JITDylib &JD, const SymbolStringPtr &Name) {
286 auto QRI = QueryRegistrations.find(Val: &JD);
287 assert(QRI != QueryRegistrations.end() &&
288 "No dependencies registered for JD");
289 assert(QRI->second.count(Name) && "No dependency on Name in JD");
290 QRI->second.erase(V: Name);
291 if (QRI->second.empty())
292 QueryRegistrations.erase(I: QRI);
293}
294
295void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
296 auto I = ResolvedSymbols.find(Val: Name);
297 assert(I != ResolvedSymbols.end() &&
298 "Redundant removal of weakly-referenced symbol");
299 ResolvedSymbols.erase(I);
300 --OutstandingSymbolsCount;
301}
302
303void AsynchronousSymbolQuery::detach() {
304 ResolvedSymbols.clear();
305 OutstandingSymbolsCount = 0;
306 for (auto &[JD, Syms] : QueryRegistrations)
307 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
308 QueryRegistrations.clear();
309}
310
311ReExportsMaterializationUnit::ReExportsMaterializationUnit(
312 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
313 SymbolAliasMap Aliases)
314 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
315 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
316
317StringRef ReExportsMaterializationUnit::getName() const {
318 return "<Reexports>";
319}
320
321void ReExportsMaterializationUnit::materialize(
322 std::unique_ptr<MaterializationResponsibility> R) {
323
324 auto &ES = R->getTargetJITDylib().getExecutionSession();
325 JITDylib &TgtJD = R->getTargetJITDylib();
326 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
327
328 // Find the set of requested aliases and aliasees. Return any unrequested
329 // aliases back to the JITDylib so as to not prematurely materialize any
330 // aliasees.
331 auto RequestedSymbols = R->getRequestedSymbols();
332 SymbolAliasMap RequestedAliases;
333
334 for (auto &Name : RequestedSymbols) {
335 auto I = Aliases.find(Val: Name);
336 assert(I != Aliases.end() && "Symbol not found in aliases map?");
337 RequestedAliases[Name] = std::move(I->second);
338 Aliases.erase(I);
339 }
340
341 LLVM_DEBUG({
342 ES.runSessionLocked([&]() {
343 dbgs() << "materializing reexports: target = " << TgtJD.getName()
344 << ", source = " << SrcJD.getName() << " " << RequestedAliases
345 << "\n";
346 });
347 });
348
349 if (!Aliases.empty()) {
350 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
351 SourceJDLookupFlags))
352 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
353
354 if (Err) {
355 // FIXME: Should this be reported / treated as failure to materialize?
356 // Or should this be treated as a sanctioned bailing-out?
357 ES.reportError(Err: std::move(Err));
358 R->failMaterialization();
359 return;
360 }
361 }
362
363 // The OnResolveInfo struct will hold the aliases and responsibility for each
364 // query in the list.
365 struct OnResolveInfo {
366 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
367 SymbolAliasMap Aliases)
368 : R(std::move(R)), Aliases(std::move(Aliases)) {}
369
370 std::unique_ptr<MaterializationResponsibility> R;
371 SymbolAliasMap Aliases;
372 std::vector<SymbolDependenceGroup> SDGs;
373 };
374
375 // Build a list of queries to issue. In each round we build a query for the
376 // largest set of aliases that we can resolve without encountering a chain of
377 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
378 // query would be waiting on a symbol that it itself had to resolve. Creating
379 // a new query for each link in such a chain eliminates the possibility of
380 // deadlock. In practice chains are likely to be rare, and this algorithm will
381 // usually result in a single query to issue.
382
383 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
384 QueryInfos;
385 while (!RequestedAliases.empty()) {
386 SymbolNameSet ResponsibilitySymbols;
387 SymbolLookupSet QuerySymbols;
388 SymbolAliasMap QueryAliases;
389
390 // Collect as many aliases as we can without including a chain.
391 for (auto &[Alias, AliasInfo] : RequestedAliases) {
392 // Chain detected. Skip this symbol for this round.
393 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: AliasInfo.Aliasee) ||
394 RequestedAliases.count(Val: AliasInfo.Aliasee)))
395 continue;
396
397 ResponsibilitySymbols.insert(V: Alias);
398 QuerySymbols.add(Name: AliasInfo.Aliasee,
399 Flags: AliasInfo.AliasFlags.hasMaterializationSideEffectsOnly()
400 ? SymbolLookupFlags::WeaklyReferencedSymbol
401 : SymbolLookupFlags::RequiredSymbol);
402 QueryAliases[Alias] = std::move(AliasInfo);
403 }
404
405 // Remove the aliases collected this round from the RequestedAliases map.
406 for (auto &KV : QueryAliases)
407 RequestedAliases.erase(Val: KV.first);
408
409 assert(!QuerySymbols.empty() && "Alias cycle detected!");
410
411 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
412 if (!NewR) {
413 ES.reportError(Err: NewR.takeError());
414 R->failMaterialization();
415 return;
416 }
417
418 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
419 args: std::move(QueryAliases));
420 QueryInfos.push_back(
421 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
422 }
423
424 // Issue the queries.
425 while (!QueryInfos.empty()) {
426 auto QuerySymbols = std::move(QueryInfos.back().first);
427 auto QueryInfo = std::move(QueryInfos.back().second);
428
429 QueryInfos.pop_back();
430
431 auto RegisterDependencies = [QueryInfo,
432 &SrcJD](const SymbolDependenceMap &Deps) {
433 // If there were no materializing symbols, just bail out.
434 if (Deps.empty())
435 return;
436
437 // Otherwise the only deps should be on SrcJD.
438 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
439 "Unexpected dependencies for reexports");
440
441 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
442
443 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
444 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
445 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
446 };
447
448 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
449 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
450 if (Result) {
451 SymbolMap ResolutionMap;
452 for (auto &KV : QueryInfo->Aliases) {
453 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
454 Result->count(KV.second.Aliasee)) &&
455 "Result map missing entry?");
456 // Don't try to resolve materialization-side-effects-only symbols.
457 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
458 continue;
459
460 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
461 KV.second.AliasFlags};
462 }
463 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
464 ES.reportError(Err: std::move(Err));
465 QueryInfo->R->failMaterialization();
466 return;
467 }
468 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
469 ES.reportError(Err: std::move(Err));
470 QueryInfo->R->failMaterialization();
471 return;
472 }
473 } else {
474 ES.reportError(Err: Result.takeError());
475 QueryInfo->R->failMaterialization();
476 }
477 };
478
479 ES.lookup(K: LookupKind::Static,
480 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
481 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
482 RegisterDependencies: std::move(RegisterDependencies));
483 }
484}
485
486void ReExportsMaterializationUnit::discard(const JITDylib &JD,
487 const SymbolStringPtr &Name) {
488 assert(Aliases.count(Name) &&
489 "Symbol not covered by this MaterializationUnit");
490 Aliases.erase(Val: Name);
491}
492
493MaterializationUnit::Interface
494ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
495 SymbolFlagsMap SymbolFlags;
496 for (auto &KV : Aliases)
497 SymbolFlags[KV.first] = KV.second.AliasFlags;
498
499 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
500}
501
502Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
503 SymbolNameSet Symbols) {
504 SymbolLookupSet LookupSet(Symbols);
505 auto Flags = SourceJD.getExecutionSession().lookupFlags(
506 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
507 Symbols: SymbolLookupSet(std::move(Symbols)));
508
509 if (!Flags)
510 return Flags.takeError();
511
512 SymbolAliasMap Result;
513 for (auto &Name : Symbols) {
514 assert(Flags->count(Name) && "Missing entry in flags map");
515 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
516 }
517
518 return Result;
519}
520
521class InProgressLookupState {
522public:
523 // FIXME: Reduce the number of SymbolStringPtrs here. See
524 // https://github.com/llvm/llvm-project/issues/55576.
525
526 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
527 SymbolLookupSet LookupSet, SymbolState RequiredState)
528 : K(K), SearchOrder(std::move(SearchOrder)),
529 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
530 DefGeneratorCandidates = this->LookupSet;
531 }
532 virtual ~InProgressLookupState() = default;
533 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
534 virtual void fail(Error Err) = 0;
535
536 LookupKind K;
537 JITDylibSearchOrder SearchOrder;
538 SymbolLookupSet LookupSet;
539 SymbolState RequiredState;
540
541 size_t CurSearchOrderIndex = 0;
542 bool NewJITDylib = true;
543 SymbolLookupSet DefGeneratorCandidates;
544 SymbolLookupSet DefGeneratorNonCandidates;
545
546 enum {
547 NotInGenerator, // Not currently using a generator.
548 ResumedForGenerator, // Resumed after being auto-suspended before generator.
549 InGenerator // Currently using generator.
550 } GenState = NotInGenerator;
551 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
552};
553
554class InProgressLookupFlagsState : public InProgressLookupState {
555public:
556 InProgressLookupFlagsState(
557 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
558 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
559 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
560 SymbolState::NeverSearched),
561 OnComplete(std::move(OnComplete)) {}
562
563 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
564 auto &ES = SearchOrder.front().first->getExecutionSession();
565 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
566 }
567
568 void fail(Error Err) override { OnComplete(std::move(Err)); }
569
570private:
571 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
572};
573
574class InProgressFullLookupState : public InProgressLookupState {
575public:
576 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
577 SymbolLookupSet LookupSet,
578 SymbolState RequiredState,
579 std::shared_ptr<AsynchronousSymbolQuery> Q,
580 RegisterDependenciesFunction RegisterDependencies)
581 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
582 RequiredState),
583 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
584 }
585
586 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
587 auto &ES = SearchOrder.front().first->getExecutionSession();
588 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
589 RegisterDependencies: std::move(RegisterDependencies));
590 }
591
592 void fail(Error Err) override {
593 Q->detach();
594 Q->handleFailed(Err: std::move(Err));
595 }
596
597private:
598 std::shared_ptr<AsynchronousSymbolQuery> Q;
599 RegisterDependenciesFunction RegisterDependencies;
600};
601
602ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
603 JITDylibLookupFlags SourceJDLookupFlags,
604 SymbolPredicate Allow)
605 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
606 Allow(std::move(Allow)) {}
607
608Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
609 JITDylib &JD,
610 JITDylibLookupFlags JDLookupFlags,
611 const SymbolLookupSet &LookupSet) {
612 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
613
614 // Use lookupFlags to find the subset of symbols that match our lookup.
615 auto Flags = JD.getExecutionSession().lookupFlags(
616 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
617 if (!Flags)
618 return Flags.takeError();
619
620 // Create an alias map.
621 orc::SymbolAliasMap AliasMap;
622 for (auto &KV : *Flags)
623 if (!Allow || Allow(KV.first))
624 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
625
626 if (AliasMap.empty())
627 return Error::success();
628
629 // Define the re-exports.
630 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
631}
632
633LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
634 : IPLS(std::move(IPLS)) {}
635
636void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
637
638LookupState::LookupState() = default;
639LookupState::LookupState(LookupState &&) = default;
640LookupState &LookupState::operator=(LookupState &&) = default;
641LookupState::~LookupState() = default;
642
643void LookupState::continueLookup(Error Err) {
644 assert(IPLS && "Cannot call continueLookup on empty LookupState");
645 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
646 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
647}
648
649DefinitionGenerator::~DefinitionGenerator() {
650 std::deque<LookupState> LookupsToFail;
651 {
652 std::lock_guard<std::mutex> Lock(M);
653 std::swap(x&: PendingLookups, y&: LookupsToFail);
654 InUse = false;
655 }
656
657 for (auto &LS : LookupsToFail)
658 LS.continueLookup(Err: make_error<StringError>(
659 Args: "Query waiting on DefinitionGenerator that was destroyed",
660 Args: inconvertibleErrorCode()));
661}
662
663JITDylib::~JITDylib() {
664 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
665}
666
667Error JITDylib::clear() {
668 std::vector<ResourceTrackerSP> TrackersToRemove;
669 ES.runSessionLocked(F: [&]() {
670 assert(State != Closed && "JD is defunct");
671 for (auto &KV : TrackerSymbols)
672 TrackersToRemove.push_back(x: KV.first);
673 TrackersToRemove.push_back(x: getDefaultResourceTracker());
674 });
675
676 Error Err = Error::success();
677 for (auto &RT : TrackersToRemove)
678 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
679 return Err;
680}
681
682ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
683 return ES.runSessionLocked(F: [this] {
684 assert(State != Closed && "JD is defunct");
685 if (!DefaultTracker)
686 DefaultTracker = new ResourceTracker(this);
687 return DefaultTracker;
688 });
689}
690
691ResourceTrackerSP JITDylib::createResourceTracker() {
692 return ES.runSessionLocked(F: [this] {
693 assert(State == Open && "JD is defunct");
694 ResourceTrackerSP RT = new ResourceTracker(this);
695 return RT;
696 });
697}
698
699void JITDylib::removeGenerator(DefinitionGenerator &G) {
700 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
701 // session lock (since it may have to send errors to pending queries).
702 std::shared_ptr<DefinitionGenerator> TmpDG;
703
704 ES.runSessionLocked(F: [&] {
705 assert(State == Open && "JD is defunct");
706 auto I = llvm::find_if(Range&: DefGenerators,
707 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
708 return H.get() == &G;
709 });
710 assert(I != DefGenerators.end() && "Generator not found");
711 TmpDG = std::move(*I);
712 DefGenerators.erase(position: I);
713 });
714}
715
716Expected<SymbolFlagsMap>
717JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
718 SymbolFlagsMap SymbolFlags) {
719
720 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
721 if (FromMR.RT->isDefunct())
722 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
723
724 std::vector<NonOwningSymbolStringPtr> AddedSyms;
725 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
726
727 for (auto &[Name, Flags] : SymbolFlags) {
728 auto EntryItr = Symbols.find(Val: Name);
729
730 // If the entry already exists...
731 if (EntryItr != Symbols.end()) {
732
733 // If this is a strong definition then error out.
734 if (!Flags.isWeak()) {
735 // Remove any symbols already added.
736 for (auto &S : AddedSyms)
737 Symbols.erase(I: Symbols.find_as(Val: S));
738
739 // FIXME: Return all duplicates.
740 return make_error<DuplicateDefinition>(
741 Args: std::string(*Name), Args: "defineMaterializing operation");
742 }
743
744 // Otherwise just make a note to discard this symbol after the loop.
745 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
746 continue;
747 } else
748 EntryItr =
749 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
750
751 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
752 EntryItr->second.setState(SymbolState::Materializing);
753 }
754
755 // Remove any rejected weak definitions from the SymbolFlags map.
756 while (!RejectedWeakDefs.empty()) {
757 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
758 RejectedWeakDefs.pop_back();
759 }
760
761 return SymbolFlags;
762 });
763}
764
765Error JITDylib::replace(MaterializationResponsibility &FromMR,
766 std::unique_ptr<MaterializationUnit> MU) {
767 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
768 std::unique_ptr<MaterializationUnit> MustRunMU;
769 std::unique_ptr<MaterializationResponsibility> MustRunMR;
770
771 auto Err =
772 ES.runSessionLocked(F: [&, this]() -> Error {
773 if (FromMR.RT->isDefunct())
774 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
775
776#ifndef NDEBUG
777 for (auto &KV : MU->getSymbols()) {
778 auto SymI = Symbols.find(KV.first);
779 assert(SymI != Symbols.end() && "Replacing unknown symbol");
780 assert(SymI->second.getState() == SymbolState::Materializing &&
781 "Can not replace a symbol that ha is not materializing");
782 assert(!SymI->second.hasMaterializerAttached() &&
783 "Symbol should not have materializer attached already");
784 assert(UnmaterializedInfos.count(KV.first) == 0 &&
785 "Symbol being replaced should have no UnmaterializedInfo");
786 }
787#endif // NDEBUG
788
789 // If the tracker is defunct we need to bail out immediately.
790
791 // If any symbol has pending queries against it then we need to
792 // materialize MU immediately.
793 for (auto &KV : MU->getSymbols()) {
794 auto MII = MaterializingInfos.find(Val: KV.first);
795 if (MII != MaterializingInfos.end()) {
796 if (MII->second.hasQueriesPending()) {
797 MustRunMR = ES.createMaterializationResponsibility(
798 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
799 InitSymbol: std::move(MU->InitSymbol));
800 MustRunMU = std::move(MU);
801 return Error::success();
802 }
803 }
804 }
805
806 // Otherwise, make MU responsible for all the symbols.
807 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
808 args: FromMR.RT.get());
809 for (auto &KV : UMI->MU->getSymbols()) {
810 auto SymI = Symbols.find(Val: KV.first);
811 assert(SymI->second.getState() == SymbolState::Materializing &&
812 "Can not replace a symbol that is not materializing");
813 assert(!SymI->second.hasMaterializerAttached() &&
814 "Can not replace a symbol that has a materializer attached");
815 assert(UnmaterializedInfos.count(KV.first) == 0 &&
816 "Unexpected materializer entry in map");
817 SymI->second.setAddress(SymI->second.getAddress());
818 SymI->second.setMaterializerAttached(true);
819
820 auto &UMIEntry = UnmaterializedInfos[KV.first];
821 assert((!UMIEntry || !UMIEntry->MU) &&
822 "Replacing symbol with materializer still attached");
823 UMIEntry = UMI;
824 }
825
826 return Error::success();
827 });
828
829 if (Err)
830 return Err;
831
832 if (MustRunMU) {
833 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
834 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
835 args: std::move(MustRunMU), args: std::move(MustRunMR)));
836 } else {
837 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
838 }
839
840 return Error::success();
841}
842
843Expected<std::unique_ptr<MaterializationResponsibility>>
844JITDylib::delegate(MaterializationResponsibility &FromMR,
845 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
846
847 return ES.runSessionLocked(
848 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
849 if (FromMR.RT->isDefunct())
850 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
851
852 return ES.createMaterializationResponsibility(
853 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
854 });
855}
856
857SymbolNameSet
858JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
859 return ES.runSessionLocked(F: [&]() {
860 SymbolNameSet RequestedSymbols;
861
862 for (auto &KV : SymbolFlags) {
863 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
864 assert(Symbols.find(KV.first)->second.getState() !=
865 SymbolState::NeverSearched &&
866 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
867 "getRequestedSymbols can only be called for symbols that have "
868 "started materializing");
869 auto I = MaterializingInfos.find(Val: KV.first);
870 if (I == MaterializingInfos.end())
871 continue;
872
873 if (I->second.hasQueriesPending())
874 RequestedSymbols.insert(V: KV.first);
875 }
876
877 return RequestedSymbols;
878 });
879}
880
881Error JITDylib::resolve(MaterializationResponsibility &MR,
882 const SymbolMap &Resolved) {
883 AsynchronousSymbolQuerySet CompletedQueries;
884
885 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
886 if (MR.RT->isDefunct())
887 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
888
889 if (State != Open)
890 return make_error<StringError>(Args: "JITDylib " + getName() +
891 " is defunct",
892 Args: inconvertibleErrorCode());
893
894 struct WorklistEntry {
895 SymbolTable::iterator SymI;
896 ExecutorSymbolDef ResolvedSym;
897 };
898
899 SymbolNameSet SymbolsInErrorState;
900 std::vector<WorklistEntry> Worklist;
901 Worklist.reserve(n: Resolved.size());
902
903 // Build worklist and check for any symbols in the error state.
904 for (const auto &KV : Resolved) {
905
906 assert(!KV.second.getFlags().hasError() &&
907 "Resolution result can not have error flag set");
908
909 auto SymI = Symbols.find(Val: KV.first);
910
911 assert(SymI != Symbols.end() && "Symbol not found");
912 assert(!SymI->second.hasMaterializerAttached() &&
913 "Resolving symbol with materializer attached?");
914 assert(SymI->second.getState() == SymbolState::Materializing &&
915 "Symbol should be materializing");
916 assert(SymI->second.getAddress() == ExecutorAddr() &&
917 "Symbol has already been resolved");
918
919 if (SymI->second.getFlags().hasError())
920 SymbolsInErrorState.insert(V: KV.first);
921 else {
922 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
923 [[maybe_unused]] auto WeakOrCommon =
924 JITSymbolFlags::Weak | JITSymbolFlags::Common;
925 assert((KV.second.getFlags() & WeakOrCommon) &&
926 "Common symbols must be resolved as common or weak");
927 assert((KV.second.getFlags() & ~WeakOrCommon) ==
928 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
929 "Resolving symbol with incorrect flags");
930
931 } else
932 assert(KV.second.getFlags() == SymI->second.getFlags() &&
933 "Resolved flags should match the declared flags");
934
935 Worklist.push_back(
936 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
937 }
938 }
939
940 // If any symbols were in the error state then bail out.
941 if (!SymbolsInErrorState.empty()) {
942 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
943 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
944 return make_error<FailedToMaterialize>(
945 Args: getExecutionSession().getSymbolStringPool(),
946 Args: std::move(FailedSymbolsDepMap));
947 }
948
949 while (!Worklist.empty()) {
950 auto SymI = Worklist.back().SymI;
951 auto ResolvedSym = Worklist.back().ResolvedSym;
952 Worklist.pop_back();
953
954 auto &Name = SymI->first;
955
956 // Resolved symbols can not be weak: discard the weak flag.
957 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
958 SymI->second.setAddress(ResolvedSym.getAddress());
959 SymI->second.setFlags(ResolvedFlags);
960 SymI->second.setState(SymbolState::Resolved);
961
962 auto MII = MaterializingInfos.find(Val: Name);
963 if (MII == MaterializingInfos.end())
964 continue;
965
966 auto &MI = MII->second;
967 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
968 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
969 if (Q->isComplete())
970 CompletedQueries.insert(x: std::move(Q));
971 }
972 }
973
974 return Error::success();
975 }))
976 return Err;
977
978 // Otherwise notify all the completed queries.
979 for (auto &Q : CompletedQueries) {
980 assert(Q->isComplete() && "Q not completed");
981 Q->handleComplete(ES);
982 }
983
984 return Error::success();
985}
986
987void JITDylib::unlinkMaterializationResponsibility(
988 MaterializationResponsibility &MR) {
989 ES.runSessionLocked(F: [&]() {
990 auto I = TrackerMRs.find(Val: MR.RT.get());
991 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
992 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
993 I->second.erase(V: &MR);
994 if (I->second.empty())
995 TrackerMRs.erase(Val: MR.RT.get());
996 });
997}
998
999void JITDylib::shrinkMaterializationInfoMemory() {
1000 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1001 // memory since we may have long-lived JDs after linking is done.
1002
1003 if (UnmaterializedInfos.empty())
1004 UnmaterializedInfos.clear();
1005
1006 if (MaterializingInfos.empty())
1007 MaterializingInfos.clear();
1008}
1009
1010void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1011 bool LinkAgainstThisJITDylibFirst) {
1012 ES.runSessionLocked(F: [&]() {
1013 assert(State == Open && "JD is defunct");
1014 if (LinkAgainstThisJITDylibFirst) {
1015 LinkOrder.clear();
1016 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1017 LinkOrder.push_back(
1018 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1019 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1020 } else
1021 LinkOrder = std::move(NewLinkOrder);
1022 });
1023}
1024
1025void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1026 ES.runSessionLocked(F: [&]() {
1027 for (auto &KV : NewLinks) {
1028 // Skip elements of NewLinks that are already in the link order.
1029 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1030 continue;
1031
1032 LinkOrder.push_back(x: std::move(KV));
1033 }
1034 });
1035}
1036
1037void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1038 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1039}
1040
1041void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1042 JITDylibLookupFlags JDLookupFlags) {
1043 ES.runSessionLocked(F: [&]() {
1044 assert(State == Open && "JD is defunct");
1045 for (auto &KV : LinkOrder)
1046 if (KV.first == &OldJD) {
1047 KV = {&NewJD, JDLookupFlags};
1048 break;
1049 }
1050 });
1051}
1052
1053void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1054 ES.runSessionLocked(F: [&]() {
1055 assert(State == Open && "JD is defunct");
1056 auto I = llvm::find_if(Range&: LinkOrder,
1057 P: [&](const JITDylibSearchOrder::value_type &KV) {
1058 return KV.first == &JD;
1059 });
1060 if (I != LinkOrder.end())
1061 LinkOrder.erase(position: I);
1062 });
1063}
1064
1065Error JITDylib::remove(const SymbolNameSet &Names) {
1066 return ES.runSessionLocked(F: [&]() -> Error {
1067 assert(State == Open && "JD is defunct");
1068 using SymbolMaterializerItrPair =
1069 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1070 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1071 SymbolNameSet Missing;
1072 SymbolNameSet Materializing;
1073
1074 for (auto &Name : Names) {
1075 auto I = Symbols.find(Val: Name);
1076
1077 // Note symbol missing.
1078 if (I == Symbols.end()) {
1079 Missing.insert(V: Name);
1080 continue;
1081 }
1082
1083 // Note symbol materializing.
1084 if (I->second.getState() != SymbolState::NeverSearched &&
1085 I->second.getState() != SymbolState::Ready) {
1086 Materializing.insert(V: Name);
1087 continue;
1088 }
1089
1090 auto UMII = I->second.hasMaterializerAttached()
1091 ? UnmaterializedInfos.find(Val: Name)
1092 : UnmaterializedInfos.end();
1093 SymbolsToRemove.push_back(x: std::make_pair(x&: I, y&: UMII));
1094 }
1095
1096 // If any of the symbols are not defined, return an error.
1097 if (!Missing.empty())
1098 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1099 Args: std::move(Missing));
1100
1101 // If any of the symbols are currently materializing, return an error.
1102 if (!Materializing.empty())
1103 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1104 Args: std::move(Materializing));
1105
1106 // Remove the symbols.
1107 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1108 auto UMII = SymbolMaterializerItrPair.second;
1109
1110 // If there is a materializer attached, call discard.
1111 if (UMII != UnmaterializedInfos.end()) {
1112 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1113 UnmaterializedInfos.erase(I: UMII);
1114 }
1115
1116 auto SymI = SymbolMaterializerItrPair.first;
1117 Symbols.erase(I: SymI);
1118 }
1119
1120 shrinkMaterializationInfoMemory();
1121
1122 return Error::success();
1123 });
1124}
1125
1126void JITDylib::dump(raw_ostream &OS) {
1127 ES.runSessionLocked(F: [&, this]() {
1128 OS << "JITDylib \"" << getName() << "\" (ES: "
1129 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1130 << ", State = ";
1131 switch (State) {
1132 case Open:
1133 OS << "Open";
1134 break;
1135 case Closing:
1136 OS << "Closing";
1137 break;
1138 case Closed:
1139 OS << "Closed";
1140 break;
1141 }
1142 OS << ")\n";
1143 if (State == Closed)
1144 return;
1145 OS << "Link order: " << LinkOrder << "\n"
1146 << "Symbol table:\n";
1147
1148 // Sort symbols so we get a deterministic order and can check them in tests.
1149 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1150 for (auto &KV : Symbols)
1151 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1152 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1153 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1154
1155 for (auto &KV : SymbolsSorted) {
1156 OS << " \"" << *KV.first << "\": ";
1157 if (auto Addr = KV.second->getAddress())
1158 OS << Addr;
1159 else
1160 OS << "<not resolved> ";
1161
1162 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1163
1164 if (KV.second->hasMaterializerAttached()) {
1165 OS << " (Materializer ";
1166 auto I = UnmaterializedInfos.find(Val: KV.first);
1167 assert(I != UnmaterializedInfos.end() &&
1168 "Lazy symbol should have UnmaterializedInfo");
1169 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1170 } else
1171 OS << "\n";
1172 }
1173
1174 if (!MaterializingInfos.empty())
1175 OS << " MaterializingInfos entries:\n";
1176 for (auto &KV : MaterializingInfos) {
1177 OS << " \"" << *KV.first << "\":\n"
1178 << " " << KV.second.pendingQueries().size()
1179 << " pending queries: { ";
1180 for (const auto &Q : KV.second.pendingQueries())
1181 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1182 OS << "}\n";
1183 }
1184 });
1185}
1186
1187void JITDylib::MaterializingInfo::addQuery(
1188 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1189
1190 auto I = llvm::lower_bound(
1191 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1192 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1193 return V->getRequiredState() <= S;
1194 });
1195 PendingQueries.insert(position: I.base(), x: std::move(Q));
1196}
1197
1198void JITDylib::MaterializingInfo::removeQuery(
1199 const AsynchronousSymbolQuery &Q) {
1200 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1201 auto I = llvm::find_if(
1202 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1203 return V.get() == &Q;
1204 });
1205 if (I != PendingQueries.end())
1206 PendingQueries.erase(position: I);
1207}
1208
1209JITDylib::AsynchronousSymbolQueryList
1210JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1211 AsynchronousSymbolQueryList Result;
1212 while (!PendingQueries.empty()) {
1213 if (PendingQueries.back()->getRequiredState() > RequiredState)
1214 break;
1215
1216 Result.push_back(x: std::move(PendingQueries.back()));
1217 PendingQueries.pop_back();
1218 }
1219
1220 return Result;
1221}
1222
1223JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1224 : JITLinkDylib(std::move(Name)), ES(ES) {
1225 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1226}
1227
1228JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1229 // Note: Should be called under the session lock.
1230 assert(State != Closed && "JD is defunct");
1231
1232 SymbolNameVector SymbolsToRemove;
1233 SymbolNameVector SymbolsToFail;
1234
1235 if (&RT == DefaultTracker.get()) {
1236 SymbolNameSet TrackedSymbols;
1237 for (auto &KV : TrackerSymbols)
1238 TrackedSymbols.insert_range(R&: KV.second);
1239
1240 for (auto &KV : Symbols) {
1241 auto &Sym = KV.first;
1242 if (!TrackedSymbols.count(V: Sym))
1243 SymbolsToRemove.push_back(x: Sym);
1244 }
1245
1246 DefaultTracker.reset();
1247 } else {
1248 /// Check for a non-default tracker.
1249 auto I = TrackerSymbols.find(Val: &RT);
1250 if (I != TrackerSymbols.end()) {
1251 SymbolsToRemove = std::move(I->second);
1252 TrackerSymbols.erase(I);
1253 }
1254 // ... if not found this tracker was already defunct. Nothing to do.
1255 }
1256
1257 for (auto &Sym : SymbolsToRemove) {
1258 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1259
1260 // If there is a MaterializingInfo then collect any queries to fail.
1261 auto MII = MaterializingInfos.find(Val: Sym);
1262 if (MII != MaterializingInfos.end())
1263 SymbolsToFail.push_back(x: Sym);
1264 }
1265
1266 auto [QueriesToFail, FailedSymbols] =
1267 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1268
1269 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1270
1271 // Removed symbols should be taken out of the table altogether.
1272 for (auto &Sym : SymbolsToRemove) {
1273 auto I = Symbols.find(Val: Sym);
1274 assert(I != Symbols.end() && "Symbol not present in table");
1275
1276 // Remove Materializer if present.
1277 if (I->second.hasMaterializerAttached()) {
1278 // FIXME: Should this discard the symbols?
1279 auto J = UnmaterializedInfos.find(Val: Sym);
1280 assert(J != UnmaterializedInfos.end() &&
1281 "Symbol table indicates MU present, but no UMI record");
1282 if (J->second->MU)
1283 DefunctMUs.push_back(x: std::move(J->second->MU));
1284 UnmaterializedInfos.erase(I: J);
1285 } else {
1286 assert(!UnmaterializedInfos.count(Sym) &&
1287 "Symbol has materializer attached");
1288 }
1289
1290 Symbols.erase(I);
1291 }
1292
1293 shrinkMaterializationInfoMemory();
1294
1295 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1296 .DefunctMUs: std::move(DefunctMUs)};
1297}
1298
1299void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1300 assert(State != Closed && "JD is defunct");
1301 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1302 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1303 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1304
1305 // Update trackers for any not-yet materialized units.
1306 for (auto &KV : UnmaterializedInfos) {
1307 if (KV.second->RT == &SrcRT)
1308 KV.second->RT = &DstRT;
1309 }
1310
1311 // Update trackers for any active materialization responsibilities.
1312 {
1313 auto I = TrackerMRs.find(Val: &SrcRT);
1314 if (I != TrackerMRs.end()) {
1315 auto &SrcMRs = I->second;
1316 auto &DstMRs = TrackerMRs[&DstRT];
1317 for (auto *MR : SrcMRs)
1318 MR->RT = &DstRT;
1319 if (DstMRs.empty())
1320 DstMRs = std::move(SrcMRs);
1321 else
1322 DstMRs.insert_range(R&: SrcMRs);
1323 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1324 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1325 TrackerMRs.erase(Val: &SrcRT);
1326 }
1327 }
1328
1329 // If we're transfering to the default tracker we just need to delete the
1330 // tracked symbols for the source tracker.
1331 if (&DstRT == DefaultTracker.get()) {
1332 TrackerSymbols.erase(Val: &SrcRT);
1333 return;
1334 }
1335
1336 // If we're transferring from the default tracker we need to find all
1337 // currently untracked symbols.
1338 if (&SrcRT == DefaultTracker.get()) {
1339 assert(!TrackerSymbols.count(&SrcRT) &&
1340 "Default tracker should not appear in TrackerSymbols");
1341
1342 SymbolNameVector SymbolsToTrack;
1343
1344 SymbolNameSet CurrentlyTrackedSymbols;
1345 for (auto &KV : TrackerSymbols)
1346 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1347
1348 for (auto &KV : Symbols) {
1349 auto &Sym = KV.first;
1350 if (!CurrentlyTrackedSymbols.count(V: Sym))
1351 SymbolsToTrack.push_back(x: Sym);
1352 }
1353
1354 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1355 return;
1356 }
1357
1358 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1359
1360 // Finally if neither SrtRT or DstRT are the default tracker then
1361 // just append DstRT's tracked symbols to SrtRT's.
1362 auto SI = TrackerSymbols.find(Val: &SrcRT);
1363 if (SI == TrackerSymbols.end())
1364 return;
1365
1366 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1367 for (auto &Sym : SI->second)
1368 DstTrackedSymbols.push_back(x: std::move(Sym));
1369 TrackerSymbols.erase(I: SI);
1370}
1371
1372Error JITDylib::defineImpl(MaterializationUnit &MU) {
1373 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1374
1375 SymbolNameSet Duplicates;
1376 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1377 std::vector<SymbolStringPtr> MUDefsOverridden;
1378
1379 for (const auto &KV : MU.getSymbols()) {
1380 auto I = Symbols.find(Val: KV.first);
1381
1382 if (I != Symbols.end()) {
1383 if (KV.second.isStrong()) {
1384 if (I->second.getFlags().isStrong() ||
1385 I->second.getState() > SymbolState::NeverSearched)
1386 Duplicates.insert(V: KV.first);
1387 else {
1388 assert(I->second.getState() == SymbolState::NeverSearched &&
1389 "Overridden existing def should be in the never-searched "
1390 "state");
1391 ExistingDefsOverridden.push_back(x: KV.first);
1392 }
1393 } else
1394 MUDefsOverridden.push_back(x: KV.first);
1395 }
1396 }
1397
1398 // If there were any duplicate definitions then bail out.
1399 if (!Duplicates.empty()) {
1400 LLVM_DEBUG(
1401 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1402 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1403 Args: MU.getName().str());
1404 }
1405
1406 // Discard any overridden defs in this MU.
1407 LLVM_DEBUG({
1408 if (!MUDefsOverridden.empty())
1409 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1410 });
1411 for (auto &S : MUDefsOverridden)
1412 MU.doDiscard(JD: *this, Name: S);
1413
1414 // Discard existing overridden defs.
1415 LLVM_DEBUG({
1416 if (!ExistingDefsOverridden.empty())
1417 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1418 << "\n";
1419 });
1420 for (auto &S : ExistingDefsOverridden) {
1421
1422 auto UMII = UnmaterializedInfos.find(Val: S);
1423 assert(UMII != UnmaterializedInfos.end() &&
1424 "Overridden existing def should have an UnmaterializedInfo");
1425 UMII->second->MU->doDiscard(JD: *this, Name: S);
1426 }
1427
1428 // Finally, add the defs from this MU.
1429 for (auto &KV : MU.getSymbols()) {
1430 auto &SymEntry = Symbols[KV.first];
1431 SymEntry.setFlags(KV.second);
1432 SymEntry.setState(SymbolState::NeverSearched);
1433 SymEntry.setMaterializerAttached(true);
1434 }
1435
1436 return Error::success();
1437}
1438
1439void JITDylib::installMaterializationUnit(
1440 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1441
1442 /// defineImpl succeeded.
1443 if (&RT != DefaultTracker.get()) {
1444 auto &TS = TrackerSymbols[&RT];
1445 TS.reserve(n: TS.size() + MU->getSymbols().size());
1446 for (auto &KV : MU->getSymbols())
1447 TS.push_back(x: KV.first);
1448 }
1449
1450 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1451 for (auto &KV : UMI->MU->getSymbols())
1452 UnmaterializedInfos[KV.first] = UMI;
1453}
1454
1455void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1456 const SymbolNameSet &QuerySymbols) {
1457 for (auto &QuerySymbol : QuerySymbols) {
1458 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1459 if (MII != MaterializingInfos.end())
1460 MII->second.removeQuery(Q);
1461 }
1462}
1463
1464Platform::~Platform() = default;
1465
1466Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1467 ExecutionSession &ES,
1468 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1469
1470 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1471 Error CompoundErr = Error::success();
1472 std::mutex LookupMutex;
1473 std::condition_variable CV;
1474 uint64_t Count = InitSyms.size();
1475
1476 LLVM_DEBUG({
1477 dbgs() << "Issuing init-symbol lookup:\n";
1478 for (auto &KV : InitSyms)
1479 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1480 });
1481
1482 for (auto &KV : InitSyms) {
1483 auto *JD = KV.first;
1484 auto Names = std::move(KV.second);
1485 ES.lookup(
1486 K: LookupKind::Static,
1487 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1488 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1489 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1490 {
1491 std::lock_guard<std::mutex> Lock(LookupMutex);
1492 --Count;
1493 if (Result) {
1494 assert(!CompoundResult.count(JD) &&
1495 "Duplicate JITDylib in lookup?");
1496 CompoundResult[JD] = std::move(*Result);
1497 } else
1498 CompoundErr =
1499 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1500 }
1501 CV.notify_one();
1502 },
1503 RegisterDependencies: NoDependenciesToRegister);
1504 }
1505
1506 std::unique_lock<std::mutex> Lock(LookupMutex);
1507 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1508
1509 if (CompoundErr)
1510 return std::move(CompoundErr);
1511
1512 return std::move(CompoundResult);
1513}
1514
1515void Platform::lookupInitSymbolsAsync(
1516 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1517 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1518
1519 class TriggerOnComplete {
1520 public:
1521 using OnCompleteFn = unique_function<void(Error)>;
1522 TriggerOnComplete(OnCompleteFn OnComplete)
1523 : OnComplete(std::move(OnComplete)) {}
1524 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1525 void reportResult(Error Err) {
1526 std::lock_guard<std::mutex> Lock(ResultMutex);
1527 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1528 }
1529
1530 private:
1531 std::mutex ResultMutex;
1532 Error LookupResult{Error::success()};
1533 OnCompleteFn OnComplete;
1534 };
1535
1536 LLVM_DEBUG({
1537 dbgs() << "Issuing init-symbol lookup:\n";
1538 for (auto &KV : InitSyms)
1539 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1540 });
1541
1542 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1543
1544 for (auto &KV : InitSyms) {
1545 auto *JD = KV.first;
1546 auto Names = std::move(KV.second);
1547 ES.lookup(
1548 K: LookupKind::Static,
1549 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1550 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1551 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1552 TOC->reportResult(Err: Result.takeError());
1553 },
1554 RegisterDependencies: NoDependenciesToRegister);
1555 }
1556}
1557
1558MaterializationTask::~MaterializationTask() {
1559 // If this task wasn't run then fail materialization.
1560 if (MR)
1561 MR->failMaterialization();
1562}
1563
1564void MaterializationTask::printDescription(raw_ostream &OS) {
1565 OS << "Materialization task: " << MU->getName() << " in "
1566 << MR->getTargetJITDylib().getName();
1567}
1568
1569void MaterializationTask::run() {
1570 assert(MU && "MU should not be null");
1571 assert(MR && "MR should not be null");
1572 MU->materialize(R: std::move(MR));
1573}
1574
1575void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1576
1577void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1578
1579ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1580 : EPC(std::move(EPC)), BootstrapJD(createBareJITDylib(Name: "<bootstrap>")) {
1581 // Associated EPC and this.
1582 this->EPC->ES = this;
1583 SymbolMap BootstrapSymbols;
1584 for (auto &[Name, Ptr] : this->EPC->getBootstrapSymbolsMap())
1585 BootstrapSymbols[intern(SymName: Name)] =
1586 ExecutorSymbolDef(Ptr, JITSymbolFlags::Exported);
1587 // Can't fail: BootstrapJD is a new, empty JD and the BootstrapSymbols
1588 // variable is a map, so can't contain duplicates.
1589 cantFail(Err: BootstrapJD.define(MU: absoluteSymbols(Symbols: std::move(BootstrapSymbols))));
1590}
1591
1592ExecutionSession::~ExecutionSession() {
1593 // You must call endSession prior to destroying the session.
1594 assert(!SessionOpen &&
1595 "Session still open. Did you forget to call endSession?");
1596}
1597
1598Error ExecutionSession::endSession() {
1599 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1600
1601 WaitingOnGraph::OpRecorder *GOpRecorderToEnd = nullptr;
1602 auto JDsToRemove = runSessionLocked(F: [&] {
1603
1604#ifdef EXPENSIVE_CHECKS
1605 verifySessionState("Entering ExecutionSession::endSession");
1606#endif
1607
1608 if (SessionOpen)
1609 GOpRecorderToEnd = GOpRecorder;
1610 SessionOpen = false;
1611 return JDs;
1612 });
1613
1614 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1615
1616 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1617
1618 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1619
1620 if (GOpRecorderToEnd)
1621 GOpRecorderToEnd->recordEnd();
1622
1623 return Err;
1624}
1625
1626void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1627 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1628}
1629
1630void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1631 runSessionLocked(F: [&] {
1632 assert(!ResourceManagers.empty() && "No managers registered");
1633 if (ResourceManagers.back() == &RM)
1634 ResourceManagers.pop_back();
1635 else {
1636 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1637 assert(I != ResourceManagers.end() && "RM not registered");
1638 ResourceManagers.erase(position: I);
1639 }
1640 });
1641}
1642
1643JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1644 return runSessionLocked(F: [&, this]() -> JITDylib * {
1645 for (auto &JD : JDs)
1646 if (JD->getName() == Name)
1647 return JD.get();
1648 return nullptr;
1649 });
1650}
1651
1652JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1653 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1654 return runSessionLocked(F: [&, this]() -> JITDylib & {
1655 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1656 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1657 return *JDs.back();
1658 });
1659}
1660
1661Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1662 auto &JD = createBareJITDylib(Name);
1663 if (P)
1664 if (auto Err = P->setupJITDylib(JD))
1665 return std::move(Err);
1666 return JD;
1667}
1668
1669Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1670 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1671 runSessionLocked(F: [&] {
1672 for (auto &JD : JDsToRemove) {
1673 assert(JD->State == JITDylib::Open && "JD already closed");
1674 JD->State = JITDylib::Closing;
1675 auto I = llvm::find(Range&: JDs, Val: JD);
1676 assert(I != JDs.end() && "JD does not appear in session JDs");
1677 JDs.erase(position: I);
1678 }
1679 });
1680
1681 // Clear JITDylibs and notify the platform.
1682 Error Err = Error::success();
1683 for (auto JD : JDsToRemove) {
1684 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1685 if (P)
1686 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1687 }
1688
1689 // Set JD to closed state. Clear remaining data structures.
1690 runSessionLocked(F: [&] {
1691 for (auto &JD : JDsToRemove) {
1692 assert(JD->State == JITDylib::Closing && "JD should be closing");
1693 JD->State = JITDylib::Closed;
1694 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1695 assert(JD->UnmaterializedInfos.empty() &&
1696 "JD.UnmaterializedInfos is not empty after clear");
1697 assert(JD->MaterializingInfos.empty() &&
1698 "JD.MaterializingInfos is not empty after clear");
1699 assert(JD->TrackerSymbols.empty() &&
1700 "TrackerSymbols is not empty after clear");
1701 JD->DefGenerators.clear();
1702 JD->LinkOrder.clear();
1703 }
1704 });
1705
1706 return Err;
1707}
1708
1709Expected<std::vector<JITDylibSP>>
1710JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1711 if (JDs.empty())
1712 return std::vector<JITDylibSP>();
1713
1714 auto &ES = JDs.front()->getExecutionSession();
1715 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1716 DenseSet<JITDylib *> Visited;
1717 std::vector<JITDylibSP> Result;
1718
1719 for (auto &JD : JDs) {
1720
1721 if (JD->State != Open)
1722 return make_error<StringError>(
1723 Args: "Error building link order: " + JD->getName() + " is defunct",
1724 Args: inconvertibleErrorCode());
1725 if (Visited.count(V: JD.get()))
1726 continue;
1727
1728 SmallVector<JITDylibSP, 64> WorkStack;
1729 WorkStack.push_back(Elt: JD);
1730 Visited.insert(V: JD.get());
1731
1732 while (!WorkStack.empty()) {
1733 Result.push_back(x: std::move(WorkStack.back()));
1734 WorkStack.pop_back();
1735
1736 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1737 auto &JD = *KV.first;
1738 if (!Visited.insert(V: &JD).second)
1739 continue;
1740 WorkStack.push_back(Elt: &JD);
1741 }
1742 }
1743 }
1744 return Result;
1745 });
1746}
1747
1748Expected<std::vector<JITDylibSP>>
1749JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1750 auto Result = getDFSLinkOrder(JDs);
1751 if (Result)
1752 std::reverse(first: Result->begin(), last: Result->end());
1753 return Result;
1754}
1755
1756Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1757 return getDFSLinkOrder(JDs: {this});
1758}
1759
1760Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1761 return getReverseDFSLinkOrder(JDs: {this});
1762}
1763
1764void ExecutionSession::lookupFlags(
1765 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1766 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1767
1768 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1769 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1770 args: std::move(OnComplete)),
1771 Err: Error::success());
1772}
1773
1774Expected<SymbolFlagsMap>
1775ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1776 SymbolLookupSet LookupSet) {
1777
1778 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1779 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1780 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1781 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1782 ResultP.set_value(std::move(Result));
1783 }),
1784 Err: Error::success());
1785
1786 auto ResultF = ResultP.get_future();
1787 return ResultF.get();
1788}
1789
1790void ExecutionSession::lookup(
1791 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1792 SymbolLookupSet Symbols, SymbolState RequiredState,
1793 SymbolsResolvedCallback NotifyComplete,
1794 RegisterDependenciesFunction RegisterDependencies) {
1795
1796 LLVM_DEBUG({
1797 runSessionLocked([&]() {
1798 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1799 << " (required state: " << RequiredState << ")\n";
1800 });
1801 });
1802
1803 // lookup can be re-entered recursively if running on a single thread. Run any
1804 // outstanding MUs in case this query depends on them, otherwise this lookup
1805 // will starve waiting for a result from an MU that is stuck in the queue.
1806 dispatchOutstandingMUs();
1807
1808 auto Unresolved = std::move(Symbols);
1809 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1810 args: std::move(NotifyComplete));
1811
1812 auto IPLS = std::make_unique<InProgressFullLookupState>(
1813 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1814 args: std::move(RegisterDependencies));
1815
1816 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1817}
1818
1819Expected<SymbolMap>
1820ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1821 SymbolLookupSet Symbols, LookupKind K,
1822 SymbolState RequiredState,
1823 RegisterDependenciesFunction RegisterDependencies) {
1824#if LLVM_ENABLE_THREADS
1825 // In the threaded case we use promises to return the results.
1826 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1827
1828 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1829 PromisedResult.set_value(std::move(R));
1830 };
1831
1832#else
1833 SymbolMap Result;
1834 Error ResolutionError = Error::success();
1835
1836 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1837 ErrorAsOutParameter _(ResolutionError);
1838 if (R)
1839 Result = std::move(*R);
1840 else
1841 ResolutionError = R.takeError();
1842 };
1843#endif
1844
1845 // Perform the asynchronous lookup.
1846 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1847 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1848
1849#if LLVM_ENABLE_THREADS
1850 return PromisedResult.get_future().get();
1851#else
1852 if (ResolutionError)
1853 return std::move(ResolutionError);
1854
1855 return Result;
1856#endif
1857}
1858
1859Expected<ExecutorSymbolDef>
1860ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1861 SymbolStringPtr Name, SymbolState RequiredState) {
1862 SymbolLookupSet Names({Name});
1863
1864 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1865 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1866 assert(ResultMap->size() == 1 && "Unexpected number of results");
1867 assert(ResultMap->count(Name) && "Missing result for symbol");
1868 return std::move(ResultMap->begin()->second);
1869 } else
1870 return ResultMap.takeError();
1871}
1872
1873Expected<ExecutorSymbolDef>
1874ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1875 SymbolState RequiredState) {
1876 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1877}
1878
1879Expected<ExecutorSymbolDef>
1880ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1881 SymbolState RequiredState) {
1882 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1883}
1884
1885Error ExecutionSession::registerJITDispatchHandlers(
1886 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1887
1888 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1889 Symbols: SymbolLookupSet::fromMapKeys(
1890 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1891 if (!TagSyms)
1892 return TagSyms.takeError();
1893
1894 // Associate tag addresses with implementations.
1895 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1896
1897 // Check that no tags are being overwritten.
1898 for (auto &[TagName, TagSym] : *TagSyms) {
1899 auto TagAddr = TagSym.getAddress();
1900 if (JITDispatchHandlers.count(Val: TagAddr))
1901 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1902 " (for " + *TagName +
1903 ") already registered",
1904 Args: inconvertibleErrorCode());
1905 }
1906
1907 // At this point we're guaranteed to succeed. Install the handlers.
1908 for (auto &[TagName, TagSym] : *TagSyms) {
1909 auto TagAddr = TagSym.getAddress();
1910 auto I = WFs.find(Val: TagName);
1911 assert(I != WFs.end() && I->second &&
1912 "JITDispatchHandler implementation missing");
1913 JITDispatchHandlers[TagAddr] =
1914 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1915 LLVM_DEBUG({
1916 dbgs() << "Associated function tag \"" << *TagName << "\" ("
1917 << formatv("{0:x}", TagAddr) << ") with handler\n";
1918 });
1919 }
1920
1921 return Error::success();
1922}
1923
1924void ExecutionSession::runJITDispatchHandler(
1925 SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr,
1926 shared::WrapperFunctionBuffer ArgBytes) {
1927
1928 std::shared_ptr<JITDispatchHandlerFunction> F;
1929 {
1930 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1931 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1932 if (I != JITDispatchHandlers.end())
1933 F = I->second;
1934 }
1935
1936 if (F)
1937 (*F)(std::move(SendResult), ArgBytes.data(), ArgBytes.size());
1938 else
1939 SendResult(shared::WrapperFunctionBuffer::createOutOfBandError(
1940 Msg: ("No function registered for tag " +
1941 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1942 .str()));
1943}
1944
1945void ExecutionSession::dump(raw_ostream &OS) {
1946 runSessionLocked(F: [this, &OS]() {
1947 for (auto &JD : JDs)
1948 JD->dump(OS);
1949 });
1950}
1951
1952#ifdef EXPENSIVE_CHECKS
1953bool ExecutionSession::verifySessionState(Twine Phase) {
1954 return runSessionLocked([&]() {
1955 bool AllOk = true;
1956
1957 for (auto &JD : JDs) {
1958
1959 auto LogFailure = [&]() -> raw_fd_ostream & {
1960 auto &Stream = errs();
1961 if (AllOk)
1962 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1963 << "\n";
1964 Stream << " In JITDylib " << JD->getName() << ", ";
1965 AllOk = false;
1966 return Stream;
1967 };
1968
1969 if (JD->State != JITDylib::Open) {
1970 LogFailure()
1971 << "state is not Open, but JD is in ExecutionSession list.";
1972 }
1973
1974 // Check symbol table.
1975 // 1. If the entry state isn't resolved then check that no address has
1976 // been set.
1977 // 2. Check that if the hasMaterializerAttached flag is set then there is
1978 // an UnmaterializedInfo entry, and vice-versa.
1979 for (auto &[Sym, Entry] : JD->Symbols) {
1980 // Check that unresolved symbols have null addresses.
1981 if (Entry.getState() < SymbolState::Resolved) {
1982 if (Entry.getAddress()) {
1983 LogFailure() << "symbol " << Sym << " has state "
1984 << Entry.getState()
1985 << " (not-yet-resolved) but non-null address "
1986 << Entry.getAddress() << ".\n";
1987 }
1988 }
1989
1990 // Check that the hasMaterializerAttached flag is correct.
1991 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
1992 if (Entry.hasMaterializerAttached()) {
1993 if (UMIItr == JD->UnmaterializedInfos.end()) {
1994 LogFailure() << "symbol " << Sym
1995 << " entry claims materializer attached, but "
1996 "UnmaterializedInfos has no corresponding entry.\n";
1997 }
1998 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
1999 LogFailure()
2000 << "symbol " << Sym
2001 << " entry claims no materializer attached, but "
2002 "UnmaterializedInfos has an unexpected entry for it.\n";
2003 }
2004 }
2005
2006 // Check that every UnmaterializedInfo entry has a corresponding entry
2007 // in the Symbols table.
2008 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2009 auto SymItr = JD->Symbols.find(Sym);
2010 if (SymItr == JD->Symbols.end()) {
2011 LogFailure()
2012 << "symbol " << Sym
2013 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2014 }
2015 }
2016
2017 // Check consistency of the MaterializingInfos table.
2018 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2019
2020 auto SymItr = JD->Symbols.find(Sym);
2021 if (SymItr == JD->Symbols.end()) {
2022 // If there's no Symbols entry for this MaterializingInfos entry then
2023 // report that.
2024 LogFailure()
2025 << "symbol " << Sym
2026 << " has MaterializingInfos entry, but no Symbols entry.\n";
2027 } else {
2028 // Otherwise check consistency between Symbols and MaterializingInfos.
2029
2030 // Ready symbols should not have MaterializingInfos.
2031 if (SymItr->second.getState() == SymbolState::Ready) {
2032 LogFailure()
2033 << "symbol " << Sym
2034 << " is in Ready state, should not have MaterializingInfo.\n";
2035 }
2036
2037 // Pending queries should be for subsequent states.
2038 auto CurState = static_cast<SymbolState>(
2039 static_cast<std::underlying_type_t<SymbolState>>(
2040 SymItr->second.getState()) + 1);
2041 for (auto &Q : MII.PendingQueries) {
2042 if (Q->getRequiredState() != CurState) {
2043 if (Q->getRequiredState() > CurState)
2044 CurState = Q->getRequiredState();
2045 else
2046 LogFailure() << "symbol " << Sym
2047 << " has stale or misordered queries.\n";
2048 }
2049 }
2050 }
2051 }
2052 }
2053
2054 return AllOk;
2055 });
2056}
2057#endif // EXPENSIVE_CHECKS
2058
2059void ExecutionSession::dispatchOutstandingMUs() {
2060 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2061 while (true) {
2062 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2063 std::unique_ptr<MaterializationResponsibility>>>
2064 JMU;
2065
2066 {
2067 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2068 if (!OutstandingMUs.empty()) {
2069 JMU.emplace(args: std::move(OutstandingMUs.back()));
2070 OutstandingMUs.pop_back();
2071 }
2072 }
2073
2074 if (!JMU)
2075 break;
2076
2077 assert(JMU->first && "No MU?");
2078 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2079 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2080 args: std::move(JMU->second)));
2081 }
2082 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2083}
2084
2085Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2086 LLVM_DEBUG({
2087 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2088 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2089 });
2090 std::vector<ResourceManager *> CurrentResourceManagers;
2091
2092 JITDylib::RemoveTrackerResult R;
2093
2094 runSessionLocked(F: [&] {
2095 CurrentResourceManagers = ResourceManagers;
2096 RT.makeDefunct();
2097 R = RT.getJITDylib().IL_removeTracker(RT);
2098 });
2099
2100 // Release any defunct MaterializationUnits.
2101 R.DefunctMUs.clear();
2102
2103 Error Err = Error::success();
2104
2105 auto &JD = RT.getJITDylib();
2106 for (auto *L : reverse(C&: CurrentResourceManagers))
2107 Err = joinErrors(E1: std::move(Err),
2108 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2109
2110 for (auto &Q : R.QueriesToFail)
2111 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2112 Args&: R.FailedSymbols));
2113
2114 return Err;
2115}
2116
2117void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2118 ResourceTracker &SrcRT) {
2119 LLVM_DEBUG({
2120 dbgs() << "In " << SrcRT.getJITDylib().getName()
2121 << " transfering resources from tracker "
2122 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2123 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2124 });
2125
2126 // No-op transfers are allowed and do not invalidate the source.
2127 if (&DstRT == &SrcRT)
2128 return;
2129
2130 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2131 "Can't transfer resources between JITDylibs");
2132 runSessionLocked(F: [&]() {
2133 SrcRT.makeDefunct();
2134 auto &JD = DstRT.getJITDylib();
2135 JD.transferTracker(DstRT, SrcRT);
2136 for (auto *L : reverse(C&: ResourceManagers))
2137 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2138 SrcK: SrcRT.getKeyUnsafe());
2139 });
2140}
2141
2142void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2143 runSessionLocked(F: [&]() {
2144 LLVM_DEBUG({
2145 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2146 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2147 });
2148 if (!RT.isDefunct())
2149 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2150 SrcRT&: RT);
2151 });
2152}
2153
2154Error ExecutionSession::IL_updateCandidatesFor(
2155 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2156 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2157 return Candidates.forEachWithRemoval(
2158 Body: [&](const SymbolStringPtr &Name,
2159 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2160 /// Search for the symbol. If not found then continue without
2161 /// removal.
2162 auto SymI = JD.Symbols.find(Val: Name);
2163 if (SymI == JD.Symbols.end())
2164 return false;
2165
2166 // If this is a non-exported symbol and we're matching exported
2167 // symbols only then remove this symbol from the candidates list.
2168 //
2169 // If we're tracking non-candidates then add this to the non-candidate
2170 // list.
2171 if (!SymI->second.getFlags().isExported() &&
2172 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2173 if (NonCandidates)
2174 NonCandidates->add(Name, Flags: SymLookupFlags);
2175 return true;
2176 }
2177
2178 // If we match against a materialization-side-effects only symbol
2179 // then make sure it is weakly-referenced. Otherwise bail out with
2180 // an error.
2181 // FIXME: Use a "materialization-side-effects-only symbols must be
2182 // weakly referenced" specific error here to reduce confusion.
2183 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2184 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2185 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2186 Args: SymbolNameVector({Name}));
2187
2188 // If we matched against this symbol but it is in the error state
2189 // then bail out and treat it as a failure to materialize.
2190 if (SymI->second.getFlags().hasError()) {
2191 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2192 (*FailedSymbolsMap)[&JD] = {Name};
2193 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2194 Args: std::move(FailedSymbolsMap));
2195 }
2196
2197 // Otherwise this is a match. Remove it from the candidate set.
2198 return true;
2199 });
2200}
2201
2202void ExecutionSession::OL_resumeLookupAfterGeneration(
2203 InProgressLookupState &IPLS) {
2204
2205 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2206 "Should not be called for not-in-generator lookups");
2207 IPLS.GenState = InProgressLookupState::NotInGenerator;
2208
2209 LookupState LS;
2210
2211 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2212 IPLS.CurDefGeneratorStack.pop_back();
2213 std::lock_guard<std::mutex> Lock(DG->M);
2214
2215 // If there are no pending lookups then mark the generator as free and
2216 // return.
2217 if (DG->PendingLookups.empty()) {
2218 DG->InUse = false;
2219 return;
2220 }
2221
2222 // Otherwise resume the next lookup.
2223 LS = std::move(DG->PendingLookups.front());
2224 DG->PendingLookups.pop_front();
2225 }
2226
2227 if (LS.IPLS) {
2228 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2229 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2230 }
2231}
2232
2233void ExecutionSession::OL_applyQueryPhase1(
2234 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2235
2236 LLVM_DEBUG({
2237 dbgs() << "Entering OL_applyQueryPhase1:\n"
2238 << " Lookup kind: " << IPLS->K << "\n"
2239 << " Search order: " << IPLS->SearchOrder
2240 << ", Current index = " << IPLS->CurSearchOrderIndex
2241 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2242 << " Lookup set: " << IPLS->LookupSet << "\n"
2243 << " Definition generator candidates: "
2244 << IPLS->DefGeneratorCandidates << "\n"
2245 << " Definition generator non-candidates: "
2246 << IPLS->DefGeneratorNonCandidates << "\n";
2247 });
2248
2249 if (IPLS->GenState == InProgressLookupState::InGenerator)
2250 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2251
2252 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2253 "Lookup should not be in InGenerator state here");
2254
2255 // FIXME: We should attach the query as we go: This provides a result in a
2256 // single pass in the common case where all symbols have already reached the
2257 // required state. The query could be detached again in the 'fail' method on
2258 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2259
2260 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2261
2262 // If we've been handed an error or received one back from a generator then
2263 // fail the query. We don't need to unlink: At this stage the query hasn't
2264 // actually been lodged.
2265 if (Err)
2266 return IPLS->fail(Err: std::move(Err));
2267
2268 // Get the next JITDylib and lookup flags.
2269 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2270 auto &JD = *KV.first;
2271 auto JDLookupFlags = KV.second;
2272
2273 LLVM_DEBUG({
2274 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2275 << ") with lookup set " << IPLS->LookupSet << ":\n";
2276 });
2277
2278 // If we've just reached a new JITDylib then perform some setup.
2279 if (IPLS->NewJITDylib) {
2280 // Add any non-candidates from the last JITDylib (if any) back on to the
2281 // list of definition candidates for this JITDylib, reset definition
2282 // non-candidates to the empty set.
2283 SymbolLookupSet Tmp;
2284 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2285 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2286
2287 LLVM_DEBUG({
2288 dbgs() << " First time visiting " << JD.getName()
2289 << ", resetting candidate sets and building generator stack\n";
2290 });
2291
2292 // Build the definition generator stack for this JITDylib.
2293 runSessionLocked(F: [&] {
2294 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2295 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2296 R: reverse(C&: JD.DefGenerators));
2297 });
2298
2299 // Flag that we've done our initialization.
2300 IPLS->NewJITDylib = false;
2301 }
2302
2303 // Remove any generation candidates that are already defined (and match) in
2304 // this JITDylib.
2305 runSessionLocked(F: [&] {
2306 // Update the list of candidates (and non-candidates) for definition
2307 // generation.
2308 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2309 Err = IL_updateCandidatesFor(
2310 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2311 NonCandidates: JD.DefGenerators.empty() ? nullptr
2312 : &IPLS->DefGeneratorNonCandidates);
2313 LLVM_DEBUG({
2314 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2315 << "\n";
2316 });
2317
2318 // If this lookup was resumed after auto-suspension but all candidates
2319 // have already been generated (by some previous call to the generator)
2320 // treat the lookup as if it had completed generation.
2321 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2322 IPLS->DefGeneratorCandidates.empty())
2323 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2324 });
2325
2326 // If we encountered an error while filtering generation candidates then
2327 // bail out.
2328 if (Err)
2329 return IPLS->fail(Err: std::move(Err));
2330
2331 /// Apply any definition generators on the stack.
2332 LLVM_DEBUG({
2333 if (IPLS->CurDefGeneratorStack.empty())
2334 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2335 else if (IPLS->DefGeneratorCandidates.empty())
2336 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2337 else
2338 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2339 << " remaining generators for "
2340 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2341 });
2342 while (!IPLS->CurDefGeneratorStack.empty() &&
2343 !IPLS->DefGeneratorCandidates.empty()) {
2344 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2345
2346 if (!DG)
2347 return IPLS->fail(Err: make_error<StringError>(
2348 Args: "DefinitionGenerator removed while lookup in progress",
2349 Args: inconvertibleErrorCode()));
2350
2351 // At this point the lookup is in either the NotInGenerator state, or in
2352 // the ResumedForGenerator state.
2353 // If this lookup is in the NotInGenerator state then check whether the
2354 // generator is in use. If the generator is not in use then move the
2355 // lookup to the InGenerator state and continue. If the generator is
2356 // already in use then just add this lookup to the pending lookups list
2357 // and bail out.
2358 // If this lookup is in the ResumedForGenerator state then just move it
2359 // to InGenerator and continue.
2360 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2361 std::lock_guard<std::mutex> Lock(DG->M);
2362 if (DG->InUse) {
2363 DG->PendingLookups.push_back(x: std::move(IPLS));
2364 return;
2365 }
2366 DG->InUse = true;
2367 }
2368
2369 IPLS->GenState = InProgressLookupState::InGenerator;
2370
2371 auto K = IPLS->K;
2372 auto &LookupSet = IPLS->DefGeneratorCandidates;
2373
2374 // Run the generator. If the generator takes ownership of QA then this
2375 // will break the loop.
2376 {
2377 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2378 LookupState LS(std::move(IPLS));
2379 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2380 IPLS = std::move(LS.IPLS);
2381 }
2382
2383 // If the lookup returned then pop the generator stack and unblock the
2384 // next lookup on this generator (if any).
2385 if (IPLS)
2386 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2387
2388 // If there was an error then fail the query.
2389 if (Err) {
2390 LLVM_DEBUG({
2391 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2392 });
2393 assert(IPLS && "LS cannot be retained if error is returned");
2394 return IPLS->fail(Err: std::move(Err));
2395 }
2396
2397 // Otherwise if QA was captured then break the loop.
2398 if (!IPLS) {
2399 LLVM_DEBUG(
2400 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2401 return;
2402 }
2403
2404 // Otherwise if we're continuing around the loop then update candidates
2405 // for the next round.
2406 runSessionLocked(F: [&] {
2407 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2408 Err = IL_updateCandidatesFor(
2409 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2410 NonCandidates: JD.DefGenerators.empty() ? nullptr
2411 : &IPLS->DefGeneratorNonCandidates);
2412 });
2413
2414 // If updating candidates failed then fail the query.
2415 if (Err) {
2416 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2417 return IPLS->fail(Err: std::move(Err));
2418 }
2419 }
2420
2421 if (IPLS->DefGeneratorCandidates.empty() &&
2422 IPLS->DefGeneratorNonCandidates.empty()) {
2423 // Early out if there are no remaining symbols.
2424 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2425 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2426 break;
2427 } else {
2428 // If we get here then we've moved on to the next JITDylib with candidates
2429 // remaining.
2430 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2431 ++IPLS->CurSearchOrderIndex;
2432 IPLS->NewJITDylib = true;
2433 }
2434 }
2435
2436 // Remove any weakly referenced candidates that could not be found/generated.
2437 IPLS->DefGeneratorCandidates.remove_if(
2438 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2439 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2440 });
2441
2442 // If we get here then we've finished searching all JITDylibs.
2443 // If we matched all symbols then move to phase 2, otherwise fail the query
2444 // with a SymbolsNotFound error.
2445 if (IPLS->DefGeneratorCandidates.empty()) {
2446 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2447 IPLS->complete(IPLS: std::move(IPLS));
2448 } else {
2449 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2450 IPLS->fail(Err: make_error<SymbolsNotFound>(
2451 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2452 }
2453}
2454
2455void ExecutionSession::OL_completeLookup(
2456 std::unique_ptr<InProgressLookupState> IPLS,
2457 std::shared_ptr<AsynchronousSymbolQuery> Q,
2458 RegisterDependenciesFunction RegisterDependencies) {
2459
2460 LLVM_DEBUG({
2461 dbgs() << "Entering OL_completeLookup:\n"
2462 << " Lookup kind: " << IPLS->K << "\n"
2463 << " Search order: " << IPLS->SearchOrder
2464 << ", Current index = " << IPLS->CurSearchOrderIndex
2465 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2466 << " Lookup set: " << IPLS->LookupSet << "\n"
2467 << " Definition generator candidates: "
2468 << IPLS->DefGeneratorCandidates << "\n"
2469 << " Definition generator non-candidates: "
2470 << IPLS->DefGeneratorNonCandidates << "\n";
2471 });
2472
2473 bool QueryComplete = false;
2474 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2475
2476 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2477 for (auto &KV : IPLS->SearchOrder) {
2478 auto &JD = *KV.first;
2479 auto JDLookupFlags = KV.second;
2480 LLVM_DEBUG({
2481 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2482 << ") with lookup set " << IPLS->LookupSet << ":\n";
2483 });
2484
2485 auto Err = IPLS->LookupSet.forEachWithRemoval(
2486 Body: [&](const SymbolStringPtr &Name,
2487 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2488 LLVM_DEBUG({
2489 dbgs() << " Attempting to match \"" << Name << "\" ("
2490 << SymLookupFlags << ")... ";
2491 });
2492
2493 /// Search for the symbol. If not found then continue without
2494 /// removal.
2495 auto SymI = JD.Symbols.find(Val: Name);
2496 if (SymI == JD.Symbols.end()) {
2497 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2498 return false;
2499 }
2500
2501 // If this is a non-exported symbol and we're matching exported
2502 // symbols only then skip this symbol without removal.
2503 if (!SymI->second.getFlags().isExported() &&
2504 JDLookupFlags ==
2505 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2506 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2507 return false;
2508 }
2509
2510 // If we match against a materialization-side-effects only symbol
2511 // then make sure it is weakly-referenced. Otherwise bail out with
2512 // an error.
2513 // FIXME: Use a "materialization-side-effects-only symbols must be
2514 // weakly referenced" specific error here to reduce confusion.
2515 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2516 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2517 LLVM_DEBUG({
2518 dbgs() << "error: "
2519 "required, but symbol is has-side-effects-only\n";
2520 });
2521 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2522 Args: SymbolNameVector({Name}));
2523 }
2524
2525 // If we matched against this symbol but it is in the error state
2526 // then bail out and treat it as a failure to materialize.
2527 if (SymI->second.getFlags().hasError()) {
2528 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2529 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2530 (*FailedSymbolsMap)[&JD] = {Name};
2531 return make_error<FailedToMaterialize>(
2532 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2533 }
2534
2535 // Otherwise this is a match.
2536
2537 // If this symbol is already in the required state then notify the
2538 // query, remove the symbol and continue.
2539 if (SymI->second.getState() >= Q->getRequiredState()) {
2540 LLVM_DEBUG(dbgs()
2541 << "matched, symbol already in required state\n");
2542 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2543
2544 // If this symbol is in anything other than the Ready state then
2545 // we need to track the dependence.
2546 if (SymI->second.getState() != SymbolState::Ready)
2547 Q->addQueryDependence(JD, Name);
2548
2549 return true;
2550 }
2551
2552 // Otherwise this symbol does not yet meet the required state. Check
2553 // whether it has a materializer attached, and if so prepare to run
2554 // it.
2555 if (SymI->second.hasMaterializerAttached()) {
2556 assert(SymI->second.getAddress() == ExecutorAddr() &&
2557 "Symbol not resolved but already has address?");
2558 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2559 assert(UMII != JD.UnmaterializedInfos.end() &&
2560 "Lazy symbol should have UnmaterializedInfo");
2561
2562 auto UMI = UMII->second;
2563 assert(UMI->MU && "Materializer should not be null");
2564 assert(UMI->RT && "Tracker should not be null");
2565 LLVM_DEBUG({
2566 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2567 << " (" << UMI->MU->getName() << ")\n";
2568 });
2569
2570 // Move all symbols associated with this MaterializationUnit into
2571 // materializing state.
2572 for (auto &KV : UMI->MU->getSymbols()) {
2573 auto SymK = JD.Symbols.find(Val: KV.first);
2574 assert(SymK != JD.Symbols.end() &&
2575 "No entry for symbol covered by MaterializationUnit");
2576 SymK->second.setMaterializerAttached(false);
2577 SymK->second.setState(SymbolState::Materializing);
2578 JD.UnmaterializedInfos.erase(Val: KV.first);
2579 }
2580
2581 // Add MU to the list of MaterializationUnits to be materialized.
2582 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2583 } else
2584 LLVM_DEBUG(dbgs() << "matched, registering query");
2585
2586 // Add the query to the PendingQueries list and continue, deleting
2587 // the element from the lookup set.
2588 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2589 SymI->second.getState() != SymbolState::Ready &&
2590 "By this line the symbol should be materializing");
2591 auto &MI = JD.MaterializingInfos[Name];
2592 MI.addQuery(Q);
2593 Q->addQueryDependence(JD, Name);
2594
2595 return true;
2596 });
2597
2598 JD.shrinkMaterializationInfoMemory();
2599
2600 // Handle failure.
2601 if (Err) {
2602
2603 LLVM_DEBUG({
2604 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2605 });
2606
2607 // Detach the query.
2608 Q->detach();
2609
2610 // Replace the MUs.
2611 for (auto &KV : CollectedUMIs) {
2612 auto &JD = *KV.first;
2613 for (auto &UMI : KV.second)
2614 for (auto &KV2 : UMI->MU->getSymbols()) {
2615 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2616 "Unexpected materializer in map");
2617 auto SymI = JD.Symbols.find(Val: KV2.first);
2618 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2619 assert(SymI->second.getState() == SymbolState::Materializing &&
2620 "Can not replace symbol that is not materializing");
2621 assert(!SymI->second.hasMaterializerAttached() &&
2622 "MaterializerAttached flag should not be set");
2623 SymI->second.setMaterializerAttached(true);
2624 JD.UnmaterializedInfos[KV2.first] = UMI;
2625 }
2626 }
2627
2628 return Err;
2629 }
2630 }
2631
2632 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2633 IPLS->LookupSet.forEachWithRemoval(
2634 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2635 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2636 Q->dropSymbol(Name);
2637 return true;
2638 } else
2639 return false;
2640 });
2641
2642 if (!IPLS->LookupSet.empty()) {
2643 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2644 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2645 Args: IPLS->LookupSet.getSymbolNames());
2646 }
2647
2648 // Record whether the query completed.
2649 QueryComplete = Q->isComplete();
2650
2651 LLVM_DEBUG({
2652 dbgs() << "Query successfully "
2653 << (QueryComplete ? "completed" : "lodged") << "\n";
2654 });
2655
2656 // Move the collected MUs to the OutstandingMUs list.
2657 if (!CollectedUMIs.empty()) {
2658 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2659
2660 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2661 for (auto &KV : CollectedUMIs) {
2662 LLVM_DEBUG({
2663 auto &JD = *KV.first;
2664 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2665 << " MUs.\n";
2666 });
2667 for (auto &UMI : KV.second) {
2668 auto MR = createMaterializationResponsibility(
2669 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2670 InitSymbol: std::move(UMI->MU->InitSymbol));
2671 OutstandingMUs.push_back(
2672 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2673 }
2674 }
2675 } else
2676 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2677
2678 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2679 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2680 RegisterDependencies(Q->QueryRegistrations);
2681 } else
2682 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2683
2684 return Error::success();
2685 });
2686
2687 if (LodgingErr) {
2688 LLVM_DEBUG(dbgs() << "Failing query\n");
2689 Q->detach();
2690 Q->handleFailed(Err: std::move(LodgingErr));
2691 return;
2692 }
2693
2694 if (QueryComplete) {
2695 LLVM_DEBUG(dbgs() << "Completing query\n");
2696 Q->handleComplete(ES&: *this);
2697 }
2698
2699 dispatchOutstandingMUs();
2700}
2701
2702void ExecutionSession::OL_completeLookupFlags(
2703 std::unique_ptr<InProgressLookupState> IPLS,
2704 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2705
2706 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2707 LLVM_DEBUG({
2708 dbgs() << "Entering OL_completeLookupFlags:\n"
2709 << " Lookup kind: " << IPLS->K << "\n"
2710 << " Search order: " << IPLS->SearchOrder
2711 << ", Current index = " << IPLS->CurSearchOrderIndex
2712 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2713 << " Lookup set: " << IPLS->LookupSet << "\n"
2714 << " Definition generator candidates: "
2715 << IPLS->DefGeneratorCandidates << "\n"
2716 << " Definition generator non-candidates: "
2717 << IPLS->DefGeneratorNonCandidates << "\n";
2718 });
2719
2720 SymbolFlagsMap Result;
2721
2722 // Attempt to find flags for each symbol.
2723 for (auto &KV : IPLS->SearchOrder) {
2724 auto &JD = *KV.first;
2725 auto JDLookupFlags = KV.second;
2726 LLVM_DEBUG({
2727 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2728 << ") with lookup set " << IPLS->LookupSet << ":\n";
2729 });
2730
2731 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2732 SymbolLookupFlags SymLookupFlags) {
2733 LLVM_DEBUG({
2734 dbgs() << " Attempting to match \"" << Name << "\" ("
2735 << SymLookupFlags << ")... ";
2736 });
2737
2738 // Search for the symbol. If not found then continue without removing
2739 // from the lookup set.
2740 auto SymI = JD.Symbols.find(Val: Name);
2741 if (SymI == JD.Symbols.end()) {
2742 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2743 return false;
2744 }
2745
2746 // If this is a non-exported symbol then it doesn't match. Skip it.
2747 if (!SymI->second.getFlags().isExported() &&
2748 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2749 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2750 return false;
2751 }
2752
2753 LLVM_DEBUG({
2754 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2755 << "\n";
2756 });
2757 Result[Name] = SymI->second.getFlags();
2758 return true;
2759 });
2760 }
2761
2762 // Remove any weakly referenced symbols that haven't been resolved.
2763 IPLS->LookupSet.remove_if(
2764 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2765 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2766 });
2767
2768 if (!IPLS->LookupSet.empty()) {
2769 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2770 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2771 Args: IPLS->LookupSet.getSymbolNames());
2772 }
2773
2774 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2775 return Result;
2776 });
2777
2778 // Run the callback on the result.
2779 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2780 OnComplete(std::move(Result));
2781}
2782
2783void ExecutionSession::OL_destroyMaterializationResponsibility(
2784 MaterializationResponsibility &MR) {
2785
2786 assert(MR.SymbolFlags.empty() &&
2787 "All symbols should have been explicitly materialized or failed");
2788 MR.JD.unlinkMaterializationResponsibility(MR);
2789}
2790
2791SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2792 const MaterializationResponsibility &MR) {
2793 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2794}
2795
2796Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2797 const SymbolMap &Symbols) {
2798 LLVM_DEBUG({
2799 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2800 });
2801#ifndef NDEBUG
2802 for (auto &KV : Symbols) {
2803 auto I = MR.SymbolFlags.find(KV.first);
2804 assert(I != MR.SymbolFlags.end() &&
2805 "Resolving symbol outside this responsibility set");
2806 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2807 "Can't resolve materialization-side-effects-only symbol");
2808 if (I->second & JITSymbolFlags::Common) {
2809 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2810 assert((KV.second.getFlags() & WeakOrCommon) &&
2811 "Common symbols must be resolved as common or weak");
2812 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2813 (I->second & ~JITSymbolFlags::Common) &&
2814 "Resolving symbol with incorrect flags");
2815 } else
2816 assert(KV.second.getFlags() == I->second &&
2817 "Resolving symbol with incorrect flags");
2818 }
2819#endif
2820
2821 return MR.JD.resolve(MR, Resolved: Symbols);
2822}
2823
2824WaitingOnGraph::ExternalState
2825ExecutionSession::IL_getSymbolState(JITDylib *JD,
2826 NonOwningSymbolStringPtr Name) {
2827 if (JD->State != JITDylib::Open)
2828 return WaitingOnGraph::ExternalState::Failed;
2829
2830 auto I = JD->Symbols.find_as(Val: Name);
2831
2832 // FIXME: Can we eliminate this possibility if we support query binding?
2833 if (I == JD->Symbols.end())
2834 return WaitingOnGraph::ExternalState::Failed;
2835
2836 if (I->second.getFlags().hasError())
2837 return WaitingOnGraph::ExternalState::Failed;
2838
2839 if (I->second.getState() == SymbolState::Ready)
2840 return WaitingOnGraph::ExternalState::Ready;
2841
2842 return WaitingOnGraph::ExternalState::None;
2843}
2844
2845template <typename UpdateSymbolFn, typename UpdateQueryFn>
2846void ExecutionSession::IL_collectQueries(
2847 JITDylib::AsynchronousSymbolQuerySet &Qs,
2848 WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
2849 UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
2850
2851 for (auto &[JD, Symbols] : QualifiedSymbols) {
2852 // IL_emit and JITDylib removal are synchronized by the session lock.
2853 // Since JITDylib removal removes any contained nodes from the
2854 // WaitingOnGraph, we should be able to assert that all nodes in the
2855 // WaitingOnGraph have not been removed.
2856 assert(JD->State == JITDylib::Open &&
2857 "WaitingOnGraph includes definition in defunct JITDylib");
2858 for (auto &Symbol : Symbols) {
2859 // Update symbol table.
2860 auto I = JD->Symbols.find_as(Val: Symbol);
2861 assert(I != JD->Symbols.end() &&
2862 "Failed Symbol missing from JD symbol table");
2863 auto &Entry = I->second;
2864 UpdateSymbol(Entry);
2865
2866 // Collect queries.
2867 auto J = JD->MaterializingInfos.find_as(Val: Symbol);
2868 if (J != JD->MaterializingInfos.end()) {
2869 for (auto &Q : J->second.takeAllPendingQueries()) {
2870 UpdateQuery(*Q, *JD, Symbol, Entry);
2871 Qs.insert(x: std::move(Q));
2872 }
2873 JD->MaterializingInfos.erase(I: J);
2874 }
2875 }
2876 }
2877}
2878
2879Expected<ExecutionSession::EmitQueries>
2880ExecutionSession::IL_emit(MaterializationResponsibility &MR,
2881 WaitingOnGraph::SimplifyResult SR) {
2882
2883 if (MR.RT->isDefunct())
2884 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
2885
2886 auto &TargetJD = MR.getTargetJITDylib();
2887 if (TargetJD.State != JITDylib::Open)
2888 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
2889 " is defunct",
2890 Args: inconvertibleErrorCode());
2891
2892#ifdef EXPENSIVE_CHECKS
2893 verifySessionState("entering ExecutionSession::IL_emit");
2894#endif
2895
2896 auto ER = G.emit(SR: std::move(SR),
2897 GetExternalState: [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
2898 return IL_getSymbolState(JD, Name);
2899 });
2900
2901 EmitQueries EQ;
2902
2903 // Handle failed queries.
2904 for (auto &SN : ER.Failed)
2905 IL_collectQueries(
2906 Qs&: EQ.Failed, QualifiedSymbols&: SN->defs(),
2907 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) {
2908 E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
2909 },
2910 UpdateQuery: [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
2911 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2912 auto &FS = EQ.FailedSymsForQuery[&Q];
2913 if (!FS)
2914 FS = std::make_shared<SymbolDependenceMap>();
2915 (*FS)[&JD].insert(V: SymbolStringPtr(Name));
2916 });
2917
2918 for (auto &FQ : EQ.Failed)
2919 FQ->detach();
2920
2921 for (auto &SN : ER.Ready)
2922 IL_collectQueries(
2923 Qs&: EQ.Completed, QualifiedSymbols&: SN->defs(),
2924 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
2925 UpdateQuery: [](AsynchronousSymbolQuery &Q, JITDylib &JD,
2926 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2927 Q.notifySymbolMetRequiredState(Name: SymbolStringPtr(Name), Sym: E.getSymbol());
2928 });
2929
2930 // std::erase_if is not available in C++17, and llvm::erase_if does not work
2931 // here.
2932 for (auto it = EQ.Completed.begin(), end = EQ.Completed.end(); it != end;) {
2933 if ((*it)->isComplete()) {
2934 ++it;
2935 } else {
2936 it = EQ.Completed.erase(position: it);
2937 }
2938 }
2939
2940#ifdef EXPENSIVE_CHECKS
2941 verifySessionState("exiting ExecutionSession::IL_emit");
2942#endif
2943
2944 return std::move(EQ);
2945}
2946
2947Error ExecutionSession::OL_notifyEmitted(
2948 MaterializationResponsibility &MR,
2949 ArrayRef<SymbolDependenceGroup> DepGroups) {
2950 LLVM_DEBUG({
2951 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
2952 << "\n";
2953 if (!DepGroups.empty()) {
2954 dbgs() << " Initial dependencies:\n";
2955 for (auto &SDG : DepGroups) {
2956 dbgs() << " Symbols: " << SDG.Symbols
2957 << ", Dependencies: " << SDG.Dependencies << "\n";
2958 }
2959 }
2960 });
2961
2962#ifndef NDEBUG
2963 SymbolNameSet Visited;
2964 for (auto &DG : DepGroups) {
2965 for (auto &Sym : DG.Symbols) {
2966 assert(MR.SymbolFlags.count(Sym) &&
2967 "DG contains dependence for symbol outside this MR");
2968 assert(Visited.insert(Sym).second &&
2969 "DG contains duplicate entries for Name");
2970 }
2971 }
2972#endif // NDEBUG
2973
2974 std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
2975 WaitingOnGraph::ContainerElementsMap Residual;
2976 {
2977 auto &JDResidual = Residual[&MR.getTargetJITDylib()];
2978 for (auto &[Name, Flags] : MR.getSymbols())
2979 JDResidual.insert(V: NonOwningSymbolStringPtr(Name));
2980
2981 for (auto &SDG : DepGroups) {
2982 WaitingOnGraph::ContainerElementsMap Defs;
2983 assert(!SDG.Symbols.empty());
2984 auto &JDDefs = Defs[&MR.getTargetJITDylib()];
2985 for (auto &Def : SDG.Symbols) {
2986 JDDefs.insert(V: NonOwningSymbolStringPtr(Def));
2987 JDResidual.erase(V: NonOwningSymbolStringPtr(Def));
2988 }
2989 WaitingOnGraph::ContainerElementsMap Deps;
2990 if (!SDG.Dependencies.empty()) {
2991 for (auto &[JD, Syms] : SDG.Dependencies) {
2992 auto &JDDeps = Deps[JD];
2993 for (auto &Dep : Syms)
2994 JDDeps.insert(V: NonOwningSymbolStringPtr(Dep));
2995 }
2996 }
2997 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
2998 args: std::move(Defs), args: std::move(Deps)));
2999 }
3000 if (!JDResidual.empty())
3001 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
3002 args: std::move(Residual), args: WaitingOnGraph::ContainerElementsMap()));
3003 }
3004
3005 auto SR = WaitingOnGraph::simplify(SNs: std::move(SNs), Rec: GOpRecorder);
3006
3007 LLVM_DEBUG({
3008 dbgs() << " Simplified dependencies:\n";
3009 for (auto &SN : SR.superNodes()) {
3010
3011 auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
3012 std::vector<JITDylib *> JDs;
3013 for (auto &[JD, _] : C)
3014 JDs.push_back(JD);
3015 llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
3016 return LHS->getName() < RHS->getName();
3017 });
3018 return JDs;
3019 };
3020
3021 auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
3022 std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
3023 llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
3024 const NonOwningSymbolStringPtr &RHS) {
3025 return *LHS < *RHS;
3026 });
3027 return Names;
3028 };
3029
3030 dbgs() << " Defs: {";
3031 for (auto *JD : SortedLibs(SN->defs())) {
3032 dbgs() << " (" << JD->getName() << ", [";
3033 for (auto &Sym : SortedNames(SN->defs()[JD]))
3034 dbgs() << " " << Sym;
3035 dbgs() << " ])";
3036 }
3037 dbgs() << " }, Deps: {";
3038 for (auto *JD : SortedLibs(SN->deps())) {
3039 dbgs() << " (" << JD->getName() << ", [";
3040 for (auto &Sym : SortedNames(SN->deps()[JD]))
3041 dbgs() << " " << Sym;
3042 dbgs() << " ])";
3043 }
3044 dbgs() << " }\n";
3045 }
3046 });
3047 auto EmitQueries =
3048 runSessionLocked(F: [&]() { return IL_emit(MR, SR: std::move(SR)); });
3049
3050 // On error bail out.
3051 if (!EmitQueries)
3052 return EmitQueries.takeError();
3053
3054 // Otherwise notify failed queries, and any updated queries that have been
3055 // completed.
3056
3057 // FIXME: Get rid of error return from notifyEmitted.
3058 SymbolDependenceMap BadDeps;
3059 {
3060 for (auto &FQ : EmitQueries->Failed) {
3061 FQ->detach();
3062 assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
3063 "Missing failed symbols for query");
3064 auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
3065 for (auto &[JD, Syms] : *FailedSyms) {
3066 auto &BadDepsForJD = BadDeps[JD];
3067 for (auto &Sym : Syms)
3068 BadDepsForJD.insert(V: Sym);
3069 }
3070 FQ->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
3071 Args: std::move(FailedSyms)));
3072 }
3073 }
3074
3075 for (auto &UQ : EmitQueries->Completed)
3076 UQ->handleComplete(ES&: *this);
3077
3078 // If there are any bad dependencies then return an error.
3079 if (!BadDeps.empty()) {
3080 SymbolNameSet BadNames;
3081 // Note: The name set calculated here is bogus: it includes all symbols in
3082 // the MR, not just the ones that failed. We want to remove the error
3083 // return path from notifyEmitted anyway, so this is just a brief
3084 // placeholder to maintain (roughly) the current error behavior.
3085 for (auto &[Name, Flags] : MR.getSymbols())
3086 BadNames.insert(V: Name);
3087 MR.SymbolFlags.clear();
3088 return make_error<UnsatisfiedSymbolDependencies>(
3089 Args: getSymbolStringPool(), Args: &MR.getTargetJITDylib(), Args: std::move(BadNames),
3090 Args: std::move(BadDeps), Args: "dependencies removed or in error state");
3091 }
3092
3093 MR.SymbolFlags.clear();
3094 return Error::success();
3095}
3096
3097Error ExecutionSession::OL_defineMaterializing(
3098 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3099
3100 LLVM_DEBUG({
3101 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3102 << NewSymbolFlags << "\n";
3103 });
3104 if (auto AcceptedDefs =
3105 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3106 // Add all newly accepted symbols to this responsibility object.
3107 for (auto &KV : *AcceptedDefs)
3108 MR.SymbolFlags.insert(KV);
3109 return Error::success();
3110 } else
3111 return AcceptedDefs.takeError();
3112}
3113
3114std::pair<JITDylib::AsynchronousSymbolQuerySet,
3115 std::shared_ptr<SymbolDependenceMap>>
3116ExecutionSession::IL_failSymbols(JITDylib &JD,
3117 const SymbolNameVector &SymbolsToFail) {
3118
3119#ifdef EXPENSIVE_CHECKS
3120 verifySessionState("entering ExecutionSession::IL_failSymbols");
3121#endif
3122
3123 // Early out in the easy case.
3124 if (SymbolsToFail.empty())
3125 return {};
3126
3127 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3128 auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
3129 auto I = FailJD->Symbols.find_as(Val: FailSym);
3130 assert(I != FailJD->Symbols.end());
3131 I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
3132 auto J = FailJD->MaterializingInfos.find_as(Val: FailSym);
3133 if (J != FailJD->MaterializingInfos.end()) {
3134 for (auto &Q : J->second.takeAllPendingQueries())
3135 FailedQueries.insert(x: std::move(Q));
3136 FailJD->MaterializingInfos.erase(I: J);
3137 }
3138 };
3139
3140 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3141
3142 {
3143 auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
3144 for (auto &Sym : SymbolsToFail) {
3145 FailedSymsForJD.insert(V: Sym);
3146 Fail(&JD, NonOwningSymbolStringPtr(Sym));
3147 }
3148 }
3149
3150 WaitingOnGraph::ContainerElementsMap ToFail;
3151 auto &JDToFail = ToFail[&JD];
3152 for (auto &Sym : SymbolsToFail)
3153 JDToFail.insert(V: NonOwningSymbolStringPtr(Sym));
3154
3155 auto FailedSNs = G.fail(Failed: ToFail, Rec: GOpRecorder);
3156
3157 for (auto &SN : FailedSNs) {
3158 for (auto &[FailJD, Defs] : SN->defs()) {
3159 auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
3160 for (auto &Def : Defs) {
3161 FailedSymsForFailJD.insert(V: SymbolStringPtr(Def));
3162 Fail(FailJD, Def);
3163 }
3164 }
3165 }
3166
3167 // Detach all failed queries.
3168 for (auto &Q : FailedQueries)
3169 Q->detach();
3170
3171#ifdef EXPENSIVE_CHECKS
3172 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3173#endif
3174
3175 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3176}
3177
3178void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3179
3180 LLVM_DEBUG({
3181 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3182 << MR.SymbolFlags << "\n";
3183 });
3184
3185 if (MR.SymbolFlags.empty())
3186 return;
3187
3188 SymbolNameVector SymbolsToFail;
3189 for (auto &[Name, Flags] : MR.SymbolFlags)
3190 SymbolsToFail.push_back(x: Name);
3191 MR.SymbolFlags.clear();
3192
3193 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3194 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3195
3196 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3197 // If the tracker is defunct then there's nothing to do here.
3198 if (MR.RT->isDefunct())
3199 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3200 std::shared_ptr<SymbolDependenceMap>>();
3201 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3202 });
3203
3204 for (auto &Q : FailedQueries) {
3205 Q->detach();
3206 Q->handleFailed(
3207 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3208 }
3209}
3210
3211Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3212 std::unique_ptr<MaterializationUnit> MU) {
3213 for (auto &KV : MU->getSymbols()) {
3214 assert(MR.SymbolFlags.count(KV.first) &&
3215 "Replacing definition outside this responsibility set");
3216 MR.SymbolFlags.erase(Val: KV.first);
3217 }
3218
3219 if (MU->getInitializerSymbol() == MR.InitSymbol)
3220 MR.InitSymbol = nullptr;
3221
3222 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3223 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3224 << "\n";
3225 }););
3226
3227 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3228}
3229
3230Expected<std::unique_ptr<MaterializationResponsibility>>
3231ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3232 const SymbolNameSet &Symbols) {
3233
3234 SymbolStringPtr DelegatedInitSymbol;
3235 SymbolFlagsMap DelegatedFlags;
3236
3237 for (auto &Name : Symbols) {
3238 auto I = MR.SymbolFlags.find(Val: Name);
3239 assert(I != MR.SymbolFlags.end() &&
3240 "Symbol is not tracked by this MaterializationResponsibility "
3241 "instance");
3242
3243 DelegatedFlags[Name] = std::move(I->second);
3244 if (Name == MR.InitSymbol)
3245 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3246
3247 MR.SymbolFlags.erase(I);
3248 }
3249
3250 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3251 InitSymbol: std::move(DelegatedInitSymbol));
3252}
3253
3254#ifndef NDEBUG
3255void ExecutionSession::dumpDispatchInfo(Task &T) {
3256 runSessionLocked([&]() {
3257 dbgs() << "Dispatching: ";
3258 T.printDescription(dbgs());
3259 dbgs() << "\n";
3260 });
3261}
3262#endif // NDEBUG
3263
3264} // End namespace orc.
3265} // End namespace llvm.
3266