1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
15#include "llvm/Support/FormatVariadic.h"
16#include "llvm/Support/MSVCErrorWorkarounds.h"
17#include "llvm/Support/raw_ostream.h"
18
19#include <condition_variable>
20#include <future>
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25namespace llvm {
26namespace orc {
27
28char ResourceTrackerDefunct::ID = 0;
29char FailedToMaterialize::ID = 0;
30char SymbolsNotFound::ID = 0;
31char SymbolsCouldNotBeRemoved::ID = 0;
32char MissingSymbolDefinitions::ID = 0;
33char UnexpectedSymbolDefinitions::ID = 0;
34char UnsatisfiedSymbolDependencies::ID = 0;
35char MaterializationTask::ID = 0;
36char LookupTask::ID = 0;
37
38RegisterDependenciesFunction NoDependenciesToRegister =
39 RegisterDependenciesFunction();
40
41void MaterializationUnit::anchor() {}
42
43ResourceTracker::ResourceTracker(JITDylibSP JD) {
44 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
45 "JITDylib must be two byte aligned");
46 JD->Retain();
47 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
48}
49
50ResourceTracker::~ResourceTracker() {
51 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
52 getJITDylib().Release();
53}
54
55Error ResourceTracker::remove() {
56 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
57}
58
59void ResourceTracker::transferTo(ResourceTracker &DstRT) {
60 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
61}
62
63void ResourceTracker::makeDefunct() {
64 uintptr_t Val = JDAndFlag.load();
65 Val |= 0x1U;
66 JDAndFlag.store(i: Val);
67}
68
69ResourceManager::~ResourceManager() = default;
70
71ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
72 : RT(std::move(RT)) {}
73
74std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
75 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
76}
77
78void ResourceTrackerDefunct::log(raw_ostream &OS) const {
79 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
80}
81
82FailedToMaterialize::FailedToMaterialize(
83 std::shared_ptr<SymbolStringPool> SSP,
84 std::shared_ptr<SymbolDependenceMap> Symbols)
85 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
86 assert(this->SSP && "String pool cannot be null");
87 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
88
89 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
90 // don't have to manually retain/release.
91 for (auto &[JD, Syms] : *this->Symbols)
92 JD->Retain();
93}
94
95FailedToMaterialize::~FailedToMaterialize() {
96 for (auto &[JD, Syms] : *Symbols)
97 JD->Release();
98}
99
100std::error_code FailedToMaterialize::convertToErrorCode() const {
101 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
102}
103
104void FailedToMaterialize::log(raw_ostream &OS) const {
105 OS << "Failed to materialize symbols: " << *Symbols;
106}
107
108UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
109 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
110 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
111 std::string Explanation)
112 : SSP(std::move(SSP)), JD(std::move(JD)),
113 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
114 Explanation(std::move(Explanation)) {}
115
116std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
117 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
118}
119
120void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
121 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
122 << ", due to unsatisfied dependencies " << BadDeps;
123 if (!Explanation.empty())
124 OS << " (" << Explanation << ")";
125}
126
127SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
128 SymbolNameSet Symbols)
129 : SSP(std::move(SSP)) {
130 llvm::append_range(C&: this->Symbols, R&: Symbols);
131 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
132}
133
134SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
135 SymbolNameVector Symbols)
136 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
137 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
138}
139
140std::error_code SymbolsNotFound::convertToErrorCode() const {
141 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
142}
143
144void SymbolsNotFound::log(raw_ostream &OS) const {
145 OS << "Symbols not found: " << Symbols;
146}
147
148SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
149 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
150 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
151 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
152}
153
154std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
155 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
156}
157
158void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
159 OS << "Symbols could not be removed: " << Symbols;
160}
161
162std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
163 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
164}
165
166void MissingSymbolDefinitions::log(raw_ostream &OS) const {
167 OS << "Missing definitions in module " << ModuleName
168 << ": " << Symbols;
169}
170
171std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
172 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
173}
174
175void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
176 OS << "Unexpected definitions in module " << ModuleName
177 << ": " << Symbols;
178}
179
180void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
181 JD->getExecutionSession().lookup(
182 K: LookupKind::Static, SearchOrder: {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
183 Symbols: SymbolLookupSet(Name), RequiredState: SymbolState::Ready,
184 NotifyComplete: [OnComplete = std::move(OnComplete)
185#ifndef NDEBUG
186 ,
187 Name = this->Name // Captured for the assert below only.
188#endif // NDEBUG
189 ](Expected<SymbolMap> Result) mutable {
190 if (Result) {
191 assert(Result->size() == 1 && "Unexpected number of results");
192 assert(Result->count(Name) &&
193 "Result does not contain expected symbol");
194 OnComplete(Result->begin()->second);
195 } else
196 OnComplete(Result.takeError());
197 },
198 RegisterDependencies: NoDependenciesToRegister);
199}
200
201AsynchronousSymbolQuery::AsynchronousSymbolQuery(
202 const SymbolLookupSet &Symbols, SymbolState RequiredState,
203 SymbolsResolvedCallback NotifyComplete)
204 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
205 assert(RequiredState >= SymbolState::Resolved &&
206 "Cannot query for a symbols that have not reached the resolve state "
207 "yet");
208
209 OutstandingSymbolsCount = Symbols.size();
210
211 for (auto &[Name, Flags] : Symbols)
212 ResolvedSymbols[Name] = ExecutorSymbolDef();
213}
214
215void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
216 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
217 auto I = ResolvedSymbols.find(Val: Name);
218 assert(I != ResolvedSymbols.end() &&
219 "Resolving symbol outside the requested set");
220 assert(I->second == ExecutorSymbolDef() &&
221 "Redundantly resolving symbol Name");
222
223 // If this is a materialization-side-effects-only symbol then drop it,
224 // otherwise update its map entry with its resolved address.
225 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
226 ResolvedSymbols.erase(I);
227 else
228 I->second = std::move(Sym);
229 --OutstandingSymbolsCount;
230}
231
232void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
233 assert(OutstandingSymbolsCount == 0 &&
234 "Symbols remain, handleComplete called prematurely");
235
236 class RunQueryCompleteTask : public Task {
237 public:
238 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
239 SymbolsResolvedCallback NotifyComplete)
240 : ResolvedSymbols(std::move(ResolvedSymbols)),
241 NotifyComplete(std::move(NotifyComplete)) {}
242 void printDescription(raw_ostream &OS) override {
243 OS << "Execute query complete callback for " << ResolvedSymbols;
244 }
245 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
246
247 private:
248 SymbolMap ResolvedSymbols;
249 SymbolsResolvedCallback NotifyComplete;
250 };
251
252 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
253 args: std::move(NotifyComplete));
254 NotifyComplete = SymbolsResolvedCallback();
255 ES.dispatchTask(T: std::move(T));
256}
257
258void AsynchronousSymbolQuery::handleFailed(Error Err) {
259 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
260 OutstandingSymbolsCount == 0 &&
261 "Query should already have been abandoned");
262 NotifyComplete(std::move(Err));
263 NotifyComplete = SymbolsResolvedCallback();
264}
265
266void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
267 SymbolStringPtr Name) {
268 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
269 (void)Added;
270 assert(Added && "Duplicate dependence notification?");
271}
272
273void AsynchronousSymbolQuery::removeQueryDependence(
274 JITDylib &JD, const SymbolStringPtr &Name) {
275 auto QRI = QueryRegistrations.find(Val: &JD);
276 assert(QRI != QueryRegistrations.end() &&
277 "No dependencies registered for JD");
278 assert(QRI->second.count(Name) && "No dependency on Name in JD");
279 QRI->second.erase(V: Name);
280 if (QRI->second.empty())
281 QueryRegistrations.erase(I: QRI);
282}
283
284void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
285 auto I = ResolvedSymbols.find(Val: Name);
286 assert(I != ResolvedSymbols.end() &&
287 "Redundant removal of weakly-referenced symbol");
288 ResolvedSymbols.erase(I);
289 --OutstandingSymbolsCount;
290}
291
292void AsynchronousSymbolQuery::detach() {
293 ResolvedSymbols.clear();
294 OutstandingSymbolsCount = 0;
295 for (auto &[JD, Syms] : QueryRegistrations)
296 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
297 QueryRegistrations.clear();
298}
299
300ReExportsMaterializationUnit::ReExportsMaterializationUnit(
301 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
302 SymbolAliasMap Aliases)
303 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
304 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
305
306StringRef ReExportsMaterializationUnit::getName() const {
307 return "<Reexports>";
308}
309
310void ReExportsMaterializationUnit::materialize(
311 std::unique_ptr<MaterializationResponsibility> R) {
312
313 auto &ES = R->getTargetJITDylib().getExecutionSession();
314 JITDylib &TgtJD = R->getTargetJITDylib();
315 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
316
317 // Find the set of requested aliases and aliasees. Return any unrequested
318 // aliases back to the JITDylib so as to not prematurely materialize any
319 // aliasees.
320 auto RequestedSymbols = R->getRequestedSymbols();
321 SymbolAliasMap RequestedAliases;
322
323 for (auto &Name : RequestedSymbols) {
324 auto I = Aliases.find(Val: Name);
325 assert(I != Aliases.end() && "Symbol not found in aliases map?");
326 RequestedAliases[Name] = std::move(I->second);
327 Aliases.erase(I);
328 }
329
330 LLVM_DEBUG({
331 ES.runSessionLocked([&]() {
332 dbgs() << "materializing reexports: target = " << TgtJD.getName()
333 << ", source = " << SrcJD.getName() << " " << RequestedAliases
334 << "\n";
335 });
336 });
337
338 if (!Aliases.empty()) {
339 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
340 SourceJDLookupFlags))
341 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
342
343 if (Err) {
344 // FIXME: Should this be reported / treated as failure to materialize?
345 // Or should this be treated as a sanctioned bailing-out?
346 ES.reportError(Err: std::move(Err));
347 R->failMaterialization();
348 return;
349 }
350 }
351
352 // The OnResolveInfo struct will hold the aliases and responsibility for each
353 // query in the list.
354 struct OnResolveInfo {
355 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
356 SymbolAliasMap Aliases)
357 : R(std::move(R)), Aliases(std::move(Aliases)) {}
358
359 std::unique_ptr<MaterializationResponsibility> R;
360 SymbolAliasMap Aliases;
361 std::vector<SymbolDependenceGroup> SDGs;
362 };
363
364 // Build a list of queries to issue. In each round we build a query for the
365 // largest set of aliases that we can resolve without encountering a chain of
366 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
367 // query would be waiting on a symbol that it itself had to resolve. Creating
368 // a new query for each link in such a chain eliminates the possibility of
369 // deadlock. In practice chains are likely to be rare, and this algorithm will
370 // usually result in a single query to issue.
371
372 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
373 QueryInfos;
374 while (!RequestedAliases.empty()) {
375 SymbolNameSet ResponsibilitySymbols;
376 SymbolLookupSet QuerySymbols;
377 SymbolAliasMap QueryAliases;
378
379 // Collect as many aliases as we can without including a chain.
380 for (auto &KV : RequestedAliases) {
381 // Chain detected. Skip this symbol for this round.
382 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: KV.second.Aliasee) ||
383 RequestedAliases.count(Val: KV.second.Aliasee)))
384 continue;
385
386 ResponsibilitySymbols.insert(V: KV.first);
387 QuerySymbols.add(Name: KV.second.Aliasee,
388 Flags: KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
389 ? SymbolLookupFlags::WeaklyReferencedSymbol
390 : SymbolLookupFlags::RequiredSymbol);
391 QueryAliases[KV.first] = std::move(KV.second);
392 }
393
394 // Remove the aliases collected this round from the RequestedAliases map.
395 for (auto &KV : QueryAliases)
396 RequestedAliases.erase(Val: KV.first);
397
398 assert(!QuerySymbols.empty() && "Alias cycle detected!");
399
400 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
401 if (!NewR) {
402 ES.reportError(Err: NewR.takeError());
403 R->failMaterialization();
404 return;
405 }
406
407 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
408 args: std::move(QueryAliases));
409 QueryInfos.push_back(
410 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
411 }
412
413 // Issue the queries.
414 while (!QueryInfos.empty()) {
415 auto QuerySymbols = std::move(QueryInfos.back().first);
416 auto QueryInfo = std::move(QueryInfos.back().second);
417
418 QueryInfos.pop_back();
419
420 auto RegisterDependencies = [QueryInfo,
421 &SrcJD](const SymbolDependenceMap &Deps) {
422 // If there were no materializing symbols, just bail out.
423 if (Deps.empty())
424 return;
425
426 // Otherwise the only deps should be on SrcJD.
427 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
428 "Unexpected dependencies for reexports");
429
430 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
431
432 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
433 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
434 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
435 };
436
437 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
438 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
439 if (Result) {
440 SymbolMap ResolutionMap;
441 for (auto &KV : QueryInfo->Aliases) {
442 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
443 Result->count(KV.second.Aliasee)) &&
444 "Result map missing entry?");
445 // Don't try to resolve materialization-side-effects-only symbols.
446 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
447 continue;
448
449 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
450 KV.second.AliasFlags};
451 }
452 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
453 ES.reportError(Err: std::move(Err));
454 QueryInfo->R->failMaterialization();
455 return;
456 }
457 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
458 ES.reportError(Err: std::move(Err));
459 QueryInfo->R->failMaterialization();
460 return;
461 }
462 } else {
463 ES.reportError(Err: Result.takeError());
464 QueryInfo->R->failMaterialization();
465 }
466 };
467
468 ES.lookup(K: LookupKind::Static,
469 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
470 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
471 RegisterDependencies: std::move(RegisterDependencies));
472 }
473}
474
475void ReExportsMaterializationUnit::discard(const JITDylib &JD,
476 const SymbolStringPtr &Name) {
477 assert(Aliases.count(Name) &&
478 "Symbol not covered by this MaterializationUnit");
479 Aliases.erase(Val: Name);
480}
481
482MaterializationUnit::Interface
483ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
484 SymbolFlagsMap SymbolFlags;
485 for (auto &KV : Aliases)
486 SymbolFlags[KV.first] = KV.second.AliasFlags;
487
488 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
489}
490
491Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
492 SymbolNameSet Symbols) {
493 SymbolLookupSet LookupSet(Symbols);
494 auto Flags = SourceJD.getExecutionSession().lookupFlags(
495 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
496 Symbols: SymbolLookupSet(std::move(Symbols)));
497
498 if (!Flags)
499 return Flags.takeError();
500
501 SymbolAliasMap Result;
502 for (auto &Name : Symbols) {
503 assert(Flags->count(Name) && "Missing entry in flags map");
504 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
505 }
506
507 return Result;
508}
509
510class InProgressLookupState {
511public:
512 // FIXME: Reduce the number of SymbolStringPtrs here. See
513 // https://github.com/llvm/llvm-project/issues/55576.
514
515 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
516 SymbolLookupSet LookupSet, SymbolState RequiredState)
517 : K(K), SearchOrder(std::move(SearchOrder)),
518 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
519 DefGeneratorCandidates = this->LookupSet;
520 }
521 virtual ~InProgressLookupState() = default;
522 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
523 virtual void fail(Error Err) = 0;
524
525 LookupKind K;
526 JITDylibSearchOrder SearchOrder;
527 SymbolLookupSet LookupSet;
528 SymbolState RequiredState;
529
530 size_t CurSearchOrderIndex = 0;
531 bool NewJITDylib = true;
532 SymbolLookupSet DefGeneratorCandidates;
533 SymbolLookupSet DefGeneratorNonCandidates;
534
535 enum {
536 NotInGenerator, // Not currently using a generator.
537 ResumedForGenerator, // Resumed after being auto-suspended before generator.
538 InGenerator // Currently using generator.
539 } GenState = NotInGenerator;
540 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
541};
542
543class InProgressLookupFlagsState : public InProgressLookupState {
544public:
545 InProgressLookupFlagsState(
546 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
547 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
548 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
549 SymbolState::NeverSearched),
550 OnComplete(std::move(OnComplete)) {}
551
552 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
553 auto &ES = SearchOrder.front().first->getExecutionSession();
554 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
555 }
556
557 void fail(Error Err) override { OnComplete(std::move(Err)); }
558
559private:
560 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
561};
562
563class InProgressFullLookupState : public InProgressLookupState {
564public:
565 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
566 SymbolLookupSet LookupSet,
567 SymbolState RequiredState,
568 std::shared_ptr<AsynchronousSymbolQuery> Q,
569 RegisterDependenciesFunction RegisterDependencies)
570 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
571 RequiredState),
572 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
573 }
574
575 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
576 auto &ES = SearchOrder.front().first->getExecutionSession();
577 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
578 RegisterDependencies: std::move(RegisterDependencies));
579 }
580
581 void fail(Error Err) override {
582 Q->detach();
583 Q->handleFailed(Err: std::move(Err));
584 }
585
586private:
587 std::shared_ptr<AsynchronousSymbolQuery> Q;
588 RegisterDependenciesFunction RegisterDependencies;
589};
590
591ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
592 JITDylibLookupFlags SourceJDLookupFlags,
593 SymbolPredicate Allow)
594 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
595 Allow(std::move(Allow)) {}
596
597Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
598 JITDylib &JD,
599 JITDylibLookupFlags JDLookupFlags,
600 const SymbolLookupSet &LookupSet) {
601 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
602
603 // Use lookupFlags to find the subset of symbols that match our lookup.
604 auto Flags = JD.getExecutionSession().lookupFlags(
605 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
606 if (!Flags)
607 return Flags.takeError();
608
609 // Create an alias map.
610 orc::SymbolAliasMap AliasMap;
611 for (auto &KV : *Flags)
612 if (!Allow || Allow(KV.first))
613 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
614
615 if (AliasMap.empty())
616 return Error::success();
617
618 // Define the re-exports.
619 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
620}
621
622LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
623 : IPLS(std::move(IPLS)) {}
624
625void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
626
627LookupState::LookupState() = default;
628LookupState::LookupState(LookupState &&) = default;
629LookupState &LookupState::operator=(LookupState &&) = default;
630LookupState::~LookupState() = default;
631
632void LookupState::continueLookup(Error Err) {
633 assert(IPLS && "Cannot call continueLookup on empty LookupState");
634 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
635 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
636}
637
638DefinitionGenerator::~DefinitionGenerator() {
639 std::deque<LookupState> LookupsToFail;
640 {
641 std::lock_guard<std::mutex> Lock(M);
642 std::swap(x&: PendingLookups, y&: LookupsToFail);
643 InUse = false;
644 }
645
646 for (auto &LS : LookupsToFail)
647 LS.continueLookup(Err: make_error<StringError>(
648 Args: "Query waiting on DefinitionGenerator that was destroyed",
649 Args: inconvertibleErrorCode()));
650}
651
652JITDylib::~JITDylib() {
653 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
654}
655
656Error JITDylib::clear() {
657 std::vector<ResourceTrackerSP> TrackersToRemove;
658 ES.runSessionLocked(F: [&]() {
659 assert(State != Closed && "JD is defunct");
660 for (auto &KV : TrackerSymbols)
661 TrackersToRemove.push_back(x: KV.first);
662 TrackersToRemove.push_back(x: getDefaultResourceTracker());
663 });
664
665 Error Err = Error::success();
666 for (auto &RT : TrackersToRemove)
667 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
668 return Err;
669}
670
671ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
672 return ES.runSessionLocked(F: [this] {
673 assert(State != Closed && "JD is defunct");
674 if (!DefaultTracker)
675 DefaultTracker = new ResourceTracker(this);
676 return DefaultTracker;
677 });
678}
679
680ResourceTrackerSP JITDylib::createResourceTracker() {
681 return ES.runSessionLocked(F: [this] {
682 assert(State == Open && "JD is defunct");
683 ResourceTrackerSP RT = new ResourceTracker(this);
684 return RT;
685 });
686}
687
688void JITDylib::removeGenerator(DefinitionGenerator &G) {
689 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
690 // session lock (since it may have to send errors to pending queries).
691 std::shared_ptr<DefinitionGenerator> TmpDG;
692
693 ES.runSessionLocked(F: [&] {
694 assert(State == Open && "JD is defunct");
695 auto I = llvm::find_if(Range&: DefGenerators,
696 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
697 return H.get() == &G;
698 });
699 assert(I != DefGenerators.end() && "Generator not found");
700 TmpDG = std::move(*I);
701 DefGenerators.erase(position: I);
702 });
703}
704
705Expected<SymbolFlagsMap>
706JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
707 SymbolFlagsMap SymbolFlags) {
708
709 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
710 if (FromMR.RT->isDefunct())
711 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
712
713 std::vector<NonOwningSymbolStringPtr> AddedSyms;
714 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
715
716 for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end();
717 SFItr != SFEnd; ++SFItr) {
718
719 auto &Name = SFItr->first;
720 auto &Flags = SFItr->second;
721
722 auto EntryItr = Symbols.find(Val: Name);
723
724 // If the entry already exists...
725 if (EntryItr != Symbols.end()) {
726
727 // If this is a strong definition then error out.
728 if (!Flags.isWeak()) {
729 // Remove any symbols already added.
730 for (auto &S : AddedSyms)
731 Symbols.erase(I: Symbols.find_as(Val: S));
732
733 // FIXME: Return all duplicates.
734 return make_error<DuplicateDefinition>(
735 Args: std::string(*Name), Args: "defineMaterializing operation");
736 }
737
738 // Otherwise just make a note to discard this symbol after the loop.
739 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
740 continue;
741 } else
742 EntryItr =
743 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
744
745 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
746 EntryItr->second.setState(SymbolState::Materializing);
747 }
748
749 // Remove any rejected weak definitions from the SymbolFlags map.
750 while (!RejectedWeakDefs.empty()) {
751 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
752 RejectedWeakDefs.pop_back();
753 }
754
755 return SymbolFlags;
756 });
757}
758
759Error JITDylib::replace(MaterializationResponsibility &FromMR,
760 std::unique_ptr<MaterializationUnit> MU) {
761 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
762 std::unique_ptr<MaterializationUnit> MustRunMU;
763 std::unique_ptr<MaterializationResponsibility> MustRunMR;
764
765 auto Err =
766 ES.runSessionLocked(F: [&, this]() -> Error {
767 if (FromMR.RT->isDefunct())
768 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
769
770#ifndef NDEBUG
771 for (auto &KV : MU->getSymbols()) {
772 auto SymI = Symbols.find(KV.first);
773 assert(SymI != Symbols.end() && "Replacing unknown symbol");
774 assert(SymI->second.getState() == SymbolState::Materializing &&
775 "Can not replace a symbol that ha is not materializing");
776 assert(!SymI->second.hasMaterializerAttached() &&
777 "Symbol should not have materializer attached already");
778 assert(UnmaterializedInfos.count(KV.first) == 0 &&
779 "Symbol being replaced should have no UnmaterializedInfo");
780 }
781#endif // NDEBUG
782
783 // If the tracker is defunct we need to bail out immediately.
784
785 // If any symbol has pending queries against it then we need to
786 // materialize MU immediately.
787 for (auto &KV : MU->getSymbols()) {
788 auto MII = MaterializingInfos.find(Val: KV.first);
789 if (MII != MaterializingInfos.end()) {
790 if (MII->second.hasQueriesPending()) {
791 MustRunMR = ES.createMaterializationResponsibility(
792 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
793 InitSymbol: std::move(MU->InitSymbol));
794 MustRunMU = std::move(MU);
795 return Error::success();
796 }
797 }
798 }
799
800 // Otherwise, make MU responsible for all the symbols.
801 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
802 args: FromMR.RT.get());
803 for (auto &KV : UMI->MU->getSymbols()) {
804 auto SymI = Symbols.find(Val: KV.first);
805 assert(SymI->second.getState() == SymbolState::Materializing &&
806 "Can not replace a symbol that is not materializing");
807 assert(!SymI->second.hasMaterializerAttached() &&
808 "Can not replace a symbol that has a materializer attached");
809 assert(UnmaterializedInfos.count(KV.first) == 0 &&
810 "Unexpected materializer entry in map");
811 SymI->second.setAddress(SymI->second.getAddress());
812 SymI->second.setMaterializerAttached(true);
813
814 auto &UMIEntry = UnmaterializedInfos[KV.first];
815 assert((!UMIEntry || !UMIEntry->MU) &&
816 "Replacing symbol with materializer still attached");
817 UMIEntry = UMI;
818 }
819
820 return Error::success();
821 });
822
823 if (Err)
824 return Err;
825
826 if (MustRunMU) {
827 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
828 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
829 args: std::move(MustRunMU), args: std::move(MustRunMR)));
830 } else {
831 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
832 }
833
834 return Error::success();
835}
836
837Expected<std::unique_ptr<MaterializationResponsibility>>
838JITDylib::delegate(MaterializationResponsibility &FromMR,
839 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
840
841 return ES.runSessionLocked(
842 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
843 if (FromMR.RT->isDefunct())
844 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
845
846 return ES.createMaterializationResponsibility(
847 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
848 });
849}
850
851SymbolNameSet
852JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
853 return ES.runSessionLocked(F: [&]() {
854 SymbolNameSet RequestedSymbols;
855
856 for (auto &KV : SymbolFlags) {
857 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
858 assert(Symbols.find(KV.first)->second.getState() !=
859 SymbolState::NeverSearched &&
860 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
861 "getRequestedSymbols can only be called for symbols that have "
862 "started materializing");
863 auto I = MaterializingInfos.find(Val: KV.first);
864 if (I == MaterializingInfos.end())
865 continue;
866
867 if (I->second.hasQueriesPending())
868 RequestedSymbols.insert(V: KV.first);
869 }
870
871 return RequestedSymbols;
872 });
873}
874
875Error JITDylib::resolve(MaterializationResponsibility &MR,
876 const SymbolMap &Resolved) {
877 AsynchronousSymbolQuerySet CompletedQueries;
878
879 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
880 if (MR.RT->isDefunct())
881 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
882
883 if (State != Open)
884 return make_error<StringError>(Args: "JITDylib " + getName() +
885 " is defunct",
886 Args: inconvertibleErrorCode());
887
888 struct WorklistEntry {
889 SymbolTable::iterator SymI;
890 ExecutorSymbolDef ResolvedSym;
891 };
892
893 SymbolNameSet SymbolsInErrorState;
894 std::vector<WorklistEntry> Worklist;
895 Worklist.reserve(n: Resolved.size());
896
897 // Build worklist and check for any symbols in the error state.
898 for (const auto &KV : Resolved) {
899
900 assert(!KV.second.getFlags().hasError() &&
901 "Resolution result can not have error flag set");
902
903 auto SymI = Symbols.find(Val: KV.first);
904
905 assert(SymI != Symbols.end() && "Symbol not found");
906 assert(!SymI->second.hasMaterializerAttached() &&
907 "Resolving symbol with materializer attached?");
908 assert(SymI->second.getState() == SymbolState::Materializing &&
909 "Symbol should be materializing");
910 assert(SymI->second.getAddress() == ExecutorAddr() &&
911 "Symbol has already been resolved");
912
913 if (SymI->second.getFlags().hasError())
914 SymbolsInErrorState.insert(V: KV.first);
915 else {
916 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
917 [[maybe_unused]] auto WeakOrCommon =
918 JITSymbolFlags::Weak | JITSymbolFlags::Common;
919 assert((KV.second.getFlags() & WeakOrCommon) &&
920 "Common symbols must be resolved as common or weak");
921 assert((KV.second.getFlags() & ~WeakOrCommon) ==
922 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
923 "Resolving symbol with incorrect flags");
924
925 } else
926 assert(KV.second.getFlags() == SymI->second.getFlags() &&
927 "Resolved flags should match the declared flags");
928
929 Worklist.push_back(
930 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
931 }
932 }
933
934 // If any symbols were in the error state then bail out.
935 if (!SymbolsInErrorState.empty()) {
936 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
937 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
938 return make_error<FailedToMaterialize>(
939 Args: getExecutionSession().getSymbolStringPool(),
940 Args: std::move(FailedSymbolsDepMap));
941 }
942
943 while (!Worklist.empty()) {
944 auto SymI = Worklist.back().SymI;
945 auto ResolvedSym = Worklist.back().ResolvedSym;
946 Worklist.pop_back();
947
948 auto &Name = SymI->first;
949
950 // Resolved symbols can not be weak: discard the weak flag.
951 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
952 SymI->second.setAddress(ResolvedSym.getAddress());
953 SymI->second.setFlags(ResolvedFlags);
954 SymI->second.setState(SymbolState::Resolved);
955
956 auto MII = MaterializingInfos.find(Val: Name);
957 if (MII == MaterializingInfos.end())
958 continue;
959
960 auto &MI = MII->second;
961 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
962 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
963 if (Q->isComplete())
964 CompletedQueries.insert(x: std::move(Q));
965 }
966 }
967
968 return Error::success();
969 }))
970 return Err;
971
972 // Otherwise notify all the completed queries.
973 for (auto &Q : CompletedQueries) {
974 assert(Q->isComplete() && "Q not completed");
975 Q->handleComplete(ES);
976 }
977
978 return Error::success();
979}
980
981void JITDylib::unlinkMaterializationResponsibility(
982 MaterializationResponsibility &MR) {
983 ES.runSessionLocked(F: [&]() {
984 auto I = TrackerMRs.find(Val: MR.RT.get());
985 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
986 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
987 I->second.erase(V: &MR);
988 if (I->second.empty())
989 TrackerMRs.erase(Val: MR.RT.get());
990 });
991}
992
993void JITDylib::shrinkMaterializationInfoMemory() {
994 // DenseMap::erase never shrinks its storage; use clear to heuristically free
995 // memory since we may have long-lived JDs after linking is done.
996
997 if (UnmaterializedInfos.empty())
998 UnmaterializedInfos.clear();
999
1000 if (MaterializingInfos.empty())
1001 MaterializingInfos.clear();
1002}
1003
1004void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1005 bool LinkAgainstThisJITDylibFirst) {
1006 ES.runSessionLocked(F: [&]() {
1007 assert(State == Open && "JD is defunct");
1008 if (LinkAgainstThisJITDylibFirst) {
1009 LinkOrder.clear();
1010 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1011 LinkOrder.push_back(
1012 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
1013 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1014 } else
1015 LinkOrder = std::move(NewLinkOrder);
1016 });
1017}
1018
1019void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1020 ES.runSessionLocked(F: [&]() {
1021 for (auto &KV : NewLinks) {
1022 // Skip elements of NewLinks that are already in the link order.
1023 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1024 continue;
1025
1026 LinkOrder.push_back(x: std::move(KV));
1027 }
1028 });
1029}
1030
1031void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1032 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1033}
1034
1035void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1036 JITDylibLookupFlags JDLookupFlags) {
1037 ES.runSessionLocked(F: [&]() {
1038 assert(State == Open && "JD is defunct");
1039 for (auto &KV : LinkOrder)
1040 if (KV.first == &OldJD) {
1041 KV = {&NewJD, JDLookupFlags};
1042 break;
1043 }
1044 });
1045}
1046
1047void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1048 ES.runSessionLocked(F: [&]() {
1049 assert(State == Open && "JD is defunct");
1050 auto I = llvm::find_if(Range&: LinkOrder,
1051 P: [&](const JITDylibSearchOrder::value_type &KV) {
1052 return KV.first == &JD;
1053 });
1054 if (I != LinkOrder.end())
1055 LinkOrder.erase(position: I);
1056 });
1057}
1058
1059Error JITDylib::remove(const SymbolNameSet &Names) {
1060 return ES.runSessionLocked(F: [&]() -> Error {
1061 assert(State == Open && "JD is defunct");
1062 using SymbolMaterializerItrPair =
1063 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1064 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1065 SymbolNameSet Missing;
1066 SymbolNameSet Materializing;
1067
1068 for (auto &Name : Names) {
1069 auto I = Symbols.find(Val: Name);
1070
1071 // Note symbol missing.
1072 if (I == Symbols.end()) {
1073 Missing.insert(V: Name);
1074 continue;
1075 }
1076
1077 // Note symbol materializing.
1078 if (I->second.getState() != SymbolState::NeverSearched &&
1079 I->second.getState() != SymbolState::Ready) {
1080 Materializing.insert(V: Name);
1081 continue;
1082 }
1083
1084 auto UMII = I->second.hasMaterializerAttached()
1085 ? UnmaterializedInfos.find(Val: Name)
1086 : UnmaterializedInfos.end();
1087 SymbolsToRemove.push_back(x: std::make_pair(x&: I, y&: UMII));
1088 }
1089
1090 // If any of the symbols are not defined, return an error.
1091 if (!Missing.empty())
1092 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1093 Args: std::move(Missing));
1094
1095 // If any of the symbols are currently materializing, return an error.
1096 if (!Materializing.empty())
1097 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1098 Args: std::move(Materializing));
1099
1100 // Remove the symbols.
1101 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1102 auto UMII = SymbolMaterializerItrPair.second;
1103
1104 // If there is a materializer attached, call discard.
1105 if (UMII != UnmaterializedInfos.end()) {
1106 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1107 UnmaterializedInfos.erase(I: UMII);
1108 }
1109
1110 auto SymI = SymbolMaterializerItrPair.first;
1111 Symbols.erase(I: SymI);
1112 }
1113
1114 shrinkMaterializationInfoMemory();
1115
1116 return Error::success();
1117 });
1118}
1119
1120void JITDylib::dump(raw_ostream &OS) {
1121 ES.runSessionLocked(F: [&, this]() {
1122 OS << "JITDylib \"" << getName() << "\" (ES: "
1123 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1124 << ", State = ";
1125 switch (State) {
1126 case Open:
1127 OS << "Open";
1128 break;
1129 case Closing:
1130 OS << "Closing";
1131 break;
1132 case Closed:
1133 OS << "Closed";
1134 break;
1135 }
1136 OS << ")\n";
1137 if (State == Closed)
1138 return;
1139 OS << "Link order: " << LinkOrder << "\n"
1140 << "Symbol table:\n";
1141
1142 // Sort symbols so we get a deterministic order and can check them in tests.
1143 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1144 for (auto &KV : Symbols)
1145 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1146 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1147 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1148
1149 for (auto &KV : SymbolsSorted) {
1150 OS << " \"" << *KV.first << "\": ";
1151 if (auto Addr = KV.second->getAddress())
1152 OS << Addr;
1153 else
1154 OS << "<not resolved> ";
1155
1156 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1157
1158 if (KV.second->hasMaterializerAttached()) {
1159 OS << " (Materializer ";
1160 auto I = UnmaterializedInfos.find(Val: KV.first);
1161 assert(I != UnmaterializedInfos.end() &&
1162 "Lazy symbol should have UnmaterializedInfo");
1163 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1164 } else
1165 OS << "\n";
1166 }
1167
1168 if (!MaterializingInfos.empty())
1169 OS << " MaterializingInfos entries:\n";
1170 for (auto &KV : MaterializingInfos) {
1171 OS << " \"" << *KV.first << "\":\n"
1172 << " " << KV.second.pendingQueries().size()
1173 << " pending queries: { ";
1174 for (const auto &Q : KV.second.pendingQueries())
1175 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1176 OS << "}\n Defining EDU: ";
1177 if (KV.second.DefiningEDU) {
1178 OS << KV.second.DefiningEDU.get() << " { ";
1179 for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
1180 OS << Name << " ";
1181 OS << "}\n";
1182 OS << " Dependencies:\n";
1183 if (!KV.second.DefiningEDU->Dependencies.empty()) {
1184 for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
1185 OS << " " << DepJD->getName() << ": [ ";
1186 for (auto &Dep : Deps)
1187 OS << Dep << " ";
1188 OS << "]\n";
1189 }
1190 } else
1191 OS << " none\n";
1192 } else
1193 OS << "none\n";
1194 OS << " Dependant EDUs:\n";
1195 if (!KV.second.DependantEDUs.empty()) {
1196 for (auto &DependantEDU : KV.second.DependantEDUs) {
1197 OS << " " << DependantEDU << ": "
1198 << DependantEDU->JD->getName() << " { ";
1199 for (auto &[Name, Flags] : DependantEDU->Symbols)
1200 OS << Name << " ";
1201 OS << "}\n";
1202 }
1203 } else
1204 OS << " none\n";
1205 assert((Symbols[KV.first].getState() != SymbolState::Ready ||
1206 (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
1207 !KV.second.DependantEDUs.empty())) &&
1208 "Stale materializing info entry");
1209 }
1210 });
1211}
1212
1213void JITDylib::MaterializingInfo::addQuery(
1214 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1215
1216 auto I = llvm::lower_bound(
1217 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1218 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1219 return V->getRequiredState() <= S;
1220 });
1221 PendingQueries.insert(position: I.base(), x: std::move(Q));
1222}
1223
1224void JITDylib::MaterializingInfo::removeQuery(
1225 const AsynchronousSymbolQuery &Q) {
1226 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1227 auto I = llvm::find_if(
1228 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1229 return V.get() == &Q;
1230 });
1231 if (I != PendingQueries.end())
1232 PendingQueries.erase(position: I);
1233}
1234
1235JITDylib::AsynchronousSymbolQueryList
1236JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1237 AsynchronousSymbolQueryList Result;
1238 while (!PendingQueries.empty()) {
1239 if (PendingQueries.back()->getRequiredState() > RequiredState)
1240 break;
1241
1242 Result.push_back(x: std::move(PendingQueries.back()));
1243 PendingQueries.pop_back();
1244 }
1245
1246 return Result;
1247}
1248
1249JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1250 : JITLinkDylib(std::move(Name)), ES(ES) {
1251 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1252}
1253
1254JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1255 // Note: Should be called under the session lock.
1256 assert(State != Closed && "JD is defunct");
1257
1258 SymbolNameVector SymbolsToRemove;
1259 SymbolNameVector SymbolsToFail;
1260
1261 if (&RT == DefaultTracker.get()) {
1262 SymbolNameSet TrackedSymbols;
1263 for (auto &KV : TrackerSymbols)
1264 TrackedSymbols.insert_range(R&: KV.second);
1265
1266 for (auto &KV : Symbols) {
1267 auto &Sym = KV.first;
1268 if (!TrackedSymbols.count(V: Sym))
1269 SymbolsToRemove.push_back(x: Sym);
1270 }
1271
1272 DefaultTracker.reset();
1273 } else {
1274 /// Check for a non-default tracker.
1275 auto I = TrackerSymbols.find(Val: &RT);
1276 if (I != TrackerSymbols.end()) {
1277 SymbolsToRemove = std::move(I->second);
1278 TrackerSymbols.erase(I);
1279 }
1280 // ... if not found this tracker was already defunct. Nothing to do.
1281 }
1282
1283 for (auto &Sym : SymbolsToRemove) {
1284 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1285
1286 // If there is a MaterializingInfo then collect any queries to fail.
1287 auto MII = MaterializingInfos.find(Val: Sym);
1288 if (MII != MaterializingInfos.end())
1289 SymbolsToFail.push_back(x: Sym);
1290 }
1291
1292 auto [QueriesToFail, FailedSymbols] =
1293 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1294
1295 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1296
1297 // Removed symbols should be taken out of the table altogether.
1298 for (auto &Sym : SymbolsToRemove) {
1299 auto I = Symbols.find(Val: Sym);
1300 assert(I != Symbols.end() && "Symbol not present in table");
1301
1302 // Remove Materializer if present.
1303 if (I->second.hasMaterializerAttached()) {
1304 // FIXME: Should this discard the symbols?
1305 auto J = UnmaterializedInfos.find(Val: Sym);
1306 assert(J != UnmaterializedInfos.end() &&
1307 "Symbol table indicates MU present, but no UMI record");
1308 if (J->second->MU)
1309 DefunctMUs.push_back(x: std::move(J->second->MU));
1310 UnmaterializedInfos.erase(I: J);
1311 } else {
1312 assert(!UnmaterializedInfos.count(Sym) &&
1313 "Symbol has materializer attached");
1314 }
1315
1316 Symbols.erase(I);
1317 }
1318
1319 shrinkMaterializationInfoMemory();
1320
1321 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1322 .DefunctMUs: std::move(DefunctMUs)};
1323}
1324
1325void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1326 assert(State != Closed && "JD is defunct");
1327 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1328 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1329 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1330
1331 // Update trackers for any not-yet materialized units.
1332 for (auto &KV : UnmaterializedInfos) {
1333 if (KV.second->RT == &SrcRT)
1334 KV.second->RT = &DstRT;
1335 }
1336
1337 // Update trackers for any active materialization responsibilities.
1338 {
1339 auto I = TrackerMRs.find(Val: &SrcRT);
1340 if (I != TrackerMRs.end()) {
1341 auto &SrcMRs = I->second;
1342 auto &DstMRs = TrackerMRs[&DstRT];
1343 for (auto *MR : SrcMRs)
1344 MR->RT = &DstRT;
1345 if (DstMRs.empty())
1346 DstMRs = std::move(SrcMRs);
1347 else
1348 DstMRs.insert_range(R&: SrcMRs);
1349 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1350 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1351 TrackerMRs.erase(Val: &SrcRT);
1352 }
1353 }
1354
1355 // If we're transfering to the default tracker we just need to delete the
1356 // tracked symbols for the source tracker.
1357 if (&DstRT == DefaultTracker.get()) {
1358 TrackerSymbols.erase(Val: &SrcRT);
1359 return;
1360 }
1361
1362 // If we're transferring from the default tracker we need to find all
1363 // currently untracked symbols.
1364 if (&SrcRT == DefaultTracker.get()) {
1365 assert(!TrackerSymbols.count(&SrcRT) &&
1366 "Default tracker should not appear in TrackerSymbols");
1367
1368 SymbolNameVector SymbolsToTrack;
1369
1370 SymbolNameSet CurrentlyTrackedSymbols;
1371 for (auto &KV : TrackerSymbols)
1372 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1373
1374 for (auto &KV : Symbols) {
1375 auto &Sym = KV.first;
1376 if (!CurrentlyTrackedSymbols.count(V: Sym))
1377 SymbolsToTrack.push_back(x: Sym);
1378 }
1379
1380 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1381 return;
1382 }
1383
1384 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1385
1386 // Finally if neither SrtRT or DstRT are the default tracker then
1387 // just append DstRT's tracked symbols to SrtRT's.
1388 auto SI = TrackerSymbols.find(Val: &SrcRT);
1389 if (SI == TrackerSymbols.end())
1390 return;
1391
1392 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1393 for (auto &Sym : SI->second)
1394 DstTrackedSymbols.push_back(x: std::move(Sym));
1395 TrackerSymbols.erase(I: SI);
1396}
1397
1398Error JITDylib::defineImpl(MaterializationUnit &MU) {
1399 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1400
1401 SymbolNameSet Duplicates;
1402 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1403 std::vector<SymbolStringPtr> MUDefsOverridden;
1404
1405 for (const auto &KV : MU.getSymbols()) {
1406 auto I = Symbols.find(Val: KV.first);
1407
1408 if (I != Symbols.end()) {
1409 if (KV.second.isStrong()) {
1410 if (I->second.getFlags().isStrong() ||
1411 I->second.getState() > SymbolState::NeverSearched)
1412 Duplicates.insert(V: KV.first);
1413 else {
1414 assert(I->second.getState() == SymbolState::NeverSearched &&
1415 "Overridden existing def should be in the never-searched "
1416 "state");
1417 ExistingDefsOverridden.push_back(x: KV.first);
1418 }
1419 } else
1420 MUDefsOverridden.push_back(x: KV.first);
1421 }
1422 }
1423
1424 // If there were any duplicate definitions then bail out.
1425 if (!Duplicates.empty()) {
1426 LLVM_DEBUG(
1427 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1428 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1429 Args: MU.getName().str());
1430 }
1431
1432 // Discard any overridden defs in this MU.
1433 LLVM_DEBUG({
1434 if (!MUDefsOverridden.empty())
1435 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1436 });
1437 for (auto &S : MUDefsOverridden)
1438 MU.doDiscard(JD: *this, Name: S);
1439
1440 // Discard existing overridden defs.
1441 LLVM_DEBUG({
1442 if (!ExistingDefsOverridden.empty())
1443 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1444 << "\n";
1445 });
1446 for (auto &S : ExistingDefsOverridden) {
1447
1448 auto UMII = UnmaterializedInfos.find(Val: S);
1449 assert(UMII != UnmaterializedInfos.end() &&
1450 "Overridden existing def should have an UnmaterializedInfo");
1451 UMII->second->MU->doDiscard(JD: *this, Name: S);
1452 }
1453
1454 // Finally, add the defs from this MU.
1455 for (auto &KV : MU.getSymbols()) {
1456 auto &SymEntry = Symbols[KV.first];
1457 SymEntry.setFlags(KV.second);
1458 SymEntry.setState(SymbolState::NeverSearched);
1459 SymEntry.setMaterializerAttached(true);
1460 }
1461
1462 return Error::success();
1463}
1464
1465void JITDylib::installMaterializationUnit(
1466 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1467
1468 /// defineImpl succeeded.
1469 if (&RT != DefaultTracker.get()) {
1470 auto &TS = TrackerSymbols[&RT];
1471 TS.reserve(n: TS.size() + MU->getSymbols().size());
1472 for (auto &KV : MU->getSymbols())
1473 TS.push_back(x: KV.first);
1474 }
1475
1476 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1477 for (auto &KV : UMI->MU->getSymbols())
1478 UnmaterializedInfos[KV.first] = UMI;
1479}
1480
1481void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1482 const SymbolNameSet &QuerySymbols) {
1483 for (auto &QuerySymbol : QuerySymbols) {
1484 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1485 if (MII != MaterializingInfos.end())
1486 MII->second.removeQuery(Q);
1487 }
1488}
1489
1490Platform::~Platform() = default;
1491
1492Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1493 ExecutionSession &ES,
1494 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1495
1496 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1497 Error CompoundErr = Error::success();
1498 std::mutex LookupMutex;
1499 std::condition_variable CV;
1500 uint64_t Count = InitSyms.size();
1501
1502 LLVM_DEBUG({
1503 dbgs() << "Issuing init-symbol lookup:\n";
1504 for (auto &KV : InitSyms)
1505 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1506 });
1507
1508 for (auto &KV : InitSyms) {
1509 auto *JD = KV.first;
1510 auto Names = std::move(KV.second);
1511 ES.lookup(
1512 K: LookupKind::Static,
1513 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1514 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1515 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1516 {
1517 std::lock_guard<std::mutex> Lock(LookupMutex);
1518 --Count;
1519 if (Result) {
1520 assert(!CompoundResult.count(JD) &&
1521 "Duplicate JITDylib in lookup?");
1522 CompoundResult[JD] = std::move(*Result);
1523 } else
1524 CompoundErr =
1525 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1526 }
1527 CV.notify_one();
1528 },
1529 RegisterDependencies: NoDependenciesToRegister);
1530 }
1531
1532 std::unique_lock<std::mutex> Lock(LookupMutex);
1533 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1534
1535 if (CompoundErr)
1536 return std::move(CompoundErr);
1537
1538 return std::move(CompoundResult);
1539}
1540
1541void Platform::lookupInitSymbolsAsync(
1542 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1543 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1544
1545 class TriggerOnComplete {
1546 public:
1547 using OnCompleteFn = unique_function<void(Error)>;
1548 TriggerOnComplete(OnCompleteFn OnComplete)
1549 : OnComplete(std::move(OnComplete)) {}
1550 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1551 void reportResult(Error Err) {
1552 std::lock_guard<std::mutex> Lock(ResultMutex);
1553 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1554 }
1555
1556 private:
1557 std::mutex ResultMutex;
1558 Error LookupResult{Error::success()};
1559 OnCompleteFn OnComplete;
1560 };
1561
1562 LLVM_DEBUG({
1563 dbgs() << "Issuing init-symbol lookup:\n";
1564 for (auto &KV : InitSyms)
1565 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1566 });
1567
1568 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1569
1570 for (auto &KV : InitSyms) {
1571 auto *JD = KV.first;
1572 auto Names = std::move(KV.second);
1573 ES.lookup(
1574 K: LookupKind::Static,
1575 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1576 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1577 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1578 TOC->reportResult(Err: Result.takeError());
1579 },
1580 RegisterDependencies: NoDependenciesToRegister);
1581 }
1582}
1583
1584MaterializationTask::~MaterializationTask() {
1585 // If this task wasn't run then fail materialization.
1586 if (MR)
1587 MR->failMaterialization();
1588}
1589
1590void MaterializationTask::printDescription(raw_ostream &OS) {
1591 OS << "Materialization task: " << MU->getName() << " in "
1592 << MR->getTargetJITDylib().getName();
1593}
1594
1595void MaterializationTask::run() {
1596 assert(MU && "MU should not be null");
1597 assert(MR && "MR should not be null");
1598 MU->materialize(R: std::move(MR));
1599}
1600
1601void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1602
1603void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1604
1605ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1606 : EPC(std::move(EPC)) {
1607 // Associated EPC and this.
1608 this->EPC->ES = this;
1609}
1610
1611ExecutionSession::~ExecutionSession() {
1612 // You must call endSession prior to destroying the session.
1613 assert(!SessionOpen &&
1614 "Session still open. Did you forget to call endSession?");
1615}
1616
1617Error ExecutionSession::endSession() {
1618 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1619
1620 auto JDsToRemove = runSessionLocked(F: [&] {
1621
1622#ifdef EXPENSIVE_CHECKS
1623 verifySessionState("Entering ExecutionSession::endSession");
1624#endif
1625
1626 SessionOpen = false;
1627 return JDs;
1628 });
1629
1630 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1631
1632 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1633
1634 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1635
1636 return Err;
1637}
1638
1639void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1640 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1641}
1642
1643void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1644 runSessionLocked(F: [&] {
1645 assert(!ResourceManagers.empty() && "No managers registered");
1646 if (ResourceManagers.back() == &RM)
1647 ResourceManagers.pop_back();
1648 else {
1649 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1650 assert(I != ResourceManagers.end() && "RM not registered");
1651 ResourceManagers.erase(position: I);
1652 }
1653 });
1654}
1655
1656JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1657 return runSessionLocked(F: [&, this]() -> JITDylib * {
1658 for (auto &JD : JDs)
1659 if (JD->getName() == Name)
1660 return JD.get();
1661 return nullptr;
1662 });
1663}
1664
1665JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1666 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1667 return runSessionLocked(F: [&, this]() -> JITDylib & {
1668 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1669 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1670 return *JDs.back();
1671 });
1672}
1673
1674Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1675 auto &JD = createBareJITDylib(Name);
1676 if (P)
1677 if (auto Err = P->setupJITDylib(JD))
1678 return std::move(Err);
1679 return JD;
1680}
1681
1682Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1683 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1684 runSessionLocked(F: [&] {
1685 for (auto &JD : JDsToRemove) {
1686 assert(JD->State == JITDylib::Open && "JD already closed");
1687 JD->State = JITDylib::Closing;
1688 auto I = llvm::find(Range&: JDs, Val: JD);
1689 assert(I != JDs.end() && "JD does not appear in session JDs");
1690 JDs.erase(position: I);
1691 }
1692 });
1693
1694 // Clear JITDylibs and notify the platform.
1695 Error Err = Error::success();
1696 for (auto JD : JDsToRemove) {
1697 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1698 if (P)
1699 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1700 }
1701
1702 // Set JD to closed state. Clear remaining data structures.
1703 runSessionLocked(F: [&] {
1704 for (auto &JD : JDsToRemove) {
1705 assert(JD->State == JITDylib::Closing && "JD should be closing");
1706 JD->State = JITDylib::Closed;
1707 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1708 assert(JD->UnmaterializedInfos.empty() &&
1709 "JD.UnmaterializedInfos is not empty after clear");
1710 assert(JD->MaterializingInfos.empty() &&
1711 "JD.MaterializingInfos is not empty after clear");
1712 assert(JD->TrackerSymbols.empty() &&
1713 "TrackerSymbols is not empty after clear");
1714 JD->DefGenerators.clear();
1715 JD->LinkOrder.clear();
1716 }
1717 });
1718
1719 return Err;
1720}
1721
1722Expected<std::vector<JITDylibSP>>
1723JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1724 if (JDs.empty())
1725 return std::vector<JITDylibSP>();
1726
1727 auto &ES = JDs.front()->getExecutionSession();
1728 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1729 DenseSet<JITDylib *> Visited;
1730 std::vector<JITDylibSP> Result;
1731
1732 for (auto &JD : JDs) {
1733
1734 if (JD->State != Open)
1735 return make_error<StringError>(
1736 Args: "Error building link order: " + JD->getName() + " is defunct",
1737 Args: inconvertibleErrorCode());
1738 if (Visited.count(V: JD.get()))
1739 continue;
1740
1741 SmallVector<JITDylibSP, 64> WorkStack;
1742 WorkStack.push_back(Elt: JD);
1743 Visited.insert(V: JD.get());
1744
1745 while (!WorkStack.empty()) {
1746 Result.push_back(x: std::move(WorkStack.back()));
1747 WorkStack.pop_back();
1748
1749 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1750 auto &JD = *KV.first;
1751 if (!Visited.insert(V: &JD).second)
1752 continue;
1753 WorkStack.push_back(Elt: &JD);
1754 }
1755 }
1756 }
1757 return Result;
1758 });
1759}
1760
1761Expected<std::vector<JITDylibSP>>
1762JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1763 auto Result = getDFSLinkOrder(JDs);
1764 if (Result)
1765 std::reverse(first: Result->begin(), last: Result->end());
1766 return Result;
1767}
1768
1769Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1770 return getDFSLinkOrder(JDs: {this});
1771}
1772
1773Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1774 return getReverseDFSLinkOrder(JDs: {this});
1775}
1776
1777void ExecutionSession::lookupFlags(
1778 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1779 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1780
1781 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1782 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1783 args: std::move(OnComplete)),
1784 Err: Error::success());
1785}
1786
1787Expected<SymbolFlagsMap>
1788ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1789 SymbolLookupSet LookupSet) {
1790
1791 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1792 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1793 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1794 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1795 ResultP.set_value(std::move(Result));
1796 }),
1797 Err: Error::success());
1798
1799 auto ResultF = ResultP.get_future();
1800 return ResultF.get();
1801}
1802
1803void ExecutionSession::lookup(
1804 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1805 SymbolLookupSet Symbols, SymbolState RequiredState,
1806 SymbolsResolvedCallback NotifyComplete,
1807 RegisterDependenciesFunction RegisterDependencies) {
1808
1809 LLVM_DEBUG({
1810 runSessionLocked([&]() {
1811 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1812 << " (required state: " << RequiredState << ")\n";
1813 });
1814 });
1815
1816 // lookup can be re-entered recursively if running on a single thread. Run any
1817 // outstanding MUs in case this query depends on them, otherwise this lookup
1818 // will starve waiting for a result from an MU that is stuck in the queue.
1819 dispatchOutstandingMUs();
1820
1821 auto Unresolved = std::move(Symbols);
1822 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1823 args: std::move(NotifyComplete));
1824
1825 auto IPLS = std::make_unique<InProgressFullLookupState>(
1826 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1827 args: std::move(RegisterDependencies));
1828
1829 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1830}
1831
1832Expected<SymbolMap>
1833ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1834 SymbolLookupSet Symbols, LookupKind K,
1835 SymbolState RequiredState,
1836 RegisterDependenciesFunction RegisterDependencies) {
1837#if LLVM_ENABLE_THREADS
1838 // In the threaded case we use promises to return the results.
1839 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1840
1841 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1842 PromisedResult.set_value(std::move(R));
1843 };
1844
1845#else
1846 SymbolMap Result;
1847 Error ResolutionError = Error::success();
1848
1849 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1850 ErrorAsOutParameter _(ResolutionError);
1851 if (R)
1852 Result = std::move(*R);
1853 else
1854 ResolutionError = R.takeError();
1855 };
1856#endif
1857
1858 // Perform the asynchronous lookup.
1859 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1860 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1861
1862#if LLVM_ENABLE_THREADS
1863 return PromisedResult.get_future().get();
1864#else
1865 if (ResolutionError)
1866 return std::move(ResolutionError);
1867
1868 return Result;
1869#endif
1870}
1871
1872Expected<ExecutorSymbolDef>
1873ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1874 SymbolStringPtr Name, SymbolState RequiredState) {
1875 SymbolLookupSet Names({Name});
1876
1877 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1878 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1879 assert(ResultMap->size() == 1 && "Unexpected number of results");
1880 assert(ResultMap->count(Name) && "Missing result for symbol");
1881 return std::move(ResultMap->begin()->second);
1882 } else
1883 return ResultMap.takeError();
1884}
1885
1886Expected<ExecutorSymbolDef>
1887ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1888 SymbolState RequiredState) {
1889 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1890}
1891
1892Expected<ExecutorSymbolDef>
1893ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1894 SymbolState RequiredState) {
1895 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1896}
1897
1898Error ExecutionSession::registerJITDispatchHandlers(
1899 JITDylib &JD, JITDispatchHandlerAssociationMap WFs) {
1900
1901 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1902 Symbols: SymbolLookupSet::fromMapKeys(
1903 M: WFs, Flags: SymbolLookupFlags::WeaklyReferencedSymbol));
1904 if (!TagSyms)
1905 return TagSyms.takeError();
1906
1907 // Associate tag addresses with implementations.
1908 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1909
1910 // Check that no tags are being overwritten.
1911 for (auto &[TagName, TagSym] : *TagSyms) {
1912 auto TagAddr = TagSym.getAddress();
1913 if (JITDispatchHandlers.count(Val: TagAddr))
1914 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1915 " (for " + *TagName +
1916 ") already registered",
1917 Args: inconvertibleErrorCode());
1918 }
1919
1920 // At this point we're guaranteed to succeed. Install the handlers.
1921 for (auto &[TagName, TagSym] : *TagSyms) {
1922 auto TagAddr = TagSym.getAddress();
1923 auto I = WFs.find(Val: TagName);
1924 assert(I != WFs.end() && I->second &&
1925 "JITDispatchHandler implementation missing");
1926 JITDispatchHandlers[TagAddr] =
1927 std::make_shared<JITDispatchHandlerFunction>(args: std::move(I->second));
1928 LLVM_DEBUG({
1929 dbgs() << "Associated function tag \"" << *TagName << "\" ("
1930 << formatv("{0:x}", TagAddr) << ") with handler\n";
1931 });
1932 }
1933
1934 return Error::success();
1935}
1936
1937void ExecutionSession::runJITDispatchHandler(SendResultFunction SendResult,
1938 ExecutorAddr HandlerFnTagAddr,
1939 ArrayRef<char> ArgBuffer) {
1940
1941 std::shared_ptr<JITDispatchHandlerFunction> F;
1942 {
1943 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1944 auto I = JITDispatchHandlers.find(Val: HandlerFnTagAddr);
1945 if (I != JITDispatchHandlers.end())
1946 F = I->second;
1947 }
1948
1949 if (F)
1950 (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size());
1951 else
1952 SendResult(shared::WrapperFunctionResult::createOutOfBandError(
1953 Msg: ("No function registered for tag " +
1954 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1955 .str()));
1956}
1957
1958void ExecutionSession::dump(raw_ostream &OS) {
1959 runSessionLocked(F: [this, &OS]() {
1960 for (auto &JD : JDs)
1961 JD->dump(OS);
1962 });
1963}
1964
1965#ifdef EXPENSIVE_CHECKS
1966bool ExecutionSession::verifySessionState(Twine Phase) {
1967 return runSessionLocked([&]() {
1968 bool AllOk = true;
1969
1970 // We'll collect these and verify them later to avoid redundant checks.
1971 DenseSet<JITDylib::EmissionDepUnit *> EDUsToCheck;
1972
1973 for (auto &JD : JDs) {
1974
1975 auto LogFailure = [&]() -> raw_fd_ostream & {
1976 auto &Stream = errs();
1977 if (AllOk)
1978 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1979 << "\n";
1980 Stream << " In JITDylib " << JD->getName() << ", ";
1981 AllOk = false;
1982 return Stream;
1983 };
1984
1985 if (JD->State != JITDylib::Open) {
1986 LogFailure()
1987 << "state is not Open, but JD is in ExecutionSession list.";
1988 }
1989
1990 // Check symbol table.
1991 // 1. If the entry state isn't resolved then check that no address has
1992 // been set.
1993 // 2. Check that if the hasMaterializerAttached flag is set then there is
1994 // an UnmaterializedInfo entry, and vice-versa.
1995 for (auto &[Sym, Entry] : JD->Symbols) {
1996 // Check that unresolved symbols have null addresses.
1997 if (Entry.getState() < SymbolState::Resolved) {
1998 if (Entry.getAddress()) {
1999 LogFailure() << "symbol " << Sym << " has state "
2000 << Entry.getState()
2001 << " (not-yet-resolved) but non-null address "
2002 << Entry.getAddress() << ".\n";
2003 }
2004 }
2005
2006 // Check that the hasMaterializerAttached flag is correct.
2007 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2008 if (Entry.hasMaterializerAttached()) {
2009 if (UMIItr == JD->UnmaterializedInfos.end()) {
2010 LogFailure() << "symbol " << Sym
2011 << " entry claims materializer attached, but "
2012 "UnmaterializedInfos has no corresponding entry.\n";
2013 }
2014 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2015 LogFailure()
2016 << "symbol " << Sym
2017 << " entry claims no materializer attached, but "
2018 "UnmaterializedInfos has an unexpected entry for it.\n";
2019 }
2020 }
2021
2022 // Check that every UnmaterializedInfo entry has a corresponding entry
2023 // in the Symbols table.
2024 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2025 auto SymItr = JD->Symbols.find(Sym);
2026 if (SymItr == JD->Symbols.end()) {
2027 LogFailure()
2028 << "symbol " << Sym
2029 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2030 }
2031 }
2032
2033 // Check consistency of the MaterializingInfos table.
2034 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2035
2036 auto SymItr = JD->Symbols.find(Sym);
2037 if (SymItr == JD->Symbols.end()) {
2038 // If there's no Symbols entry for this MaterializingInfos entry then
2039 // report that.
2040 LogFailure()
2041 << "symbol " << Sym
2042 << " has MaterializingInfos entry, but no Symbols entry.\n";
2043 } else {
2044 // Otherwise check consistency between Symbols and MaterializingInfos.
2045
2046 // Ready symbols should not have MaterializingInfos.
2047 if (SymItr->second.getState() == SymbolState::Ready) {
2048 LogFailure()
2049 << "symbol " << Sym
2050 << " is in Ready state, should not have MaterializingInfo.\n";
2051 }
2052
2053 // Pending queries should be for subsequent states.
2054 auto CurState = static_cast<SymbolState>(
2055 static_cast<std::underlying_type_t<SymbolState>>(
2056 SymItr->second.getState()) + 1);
2057 for (auto &Q : MII.PendingQueries) {
2058 if (Q->getRequiredState() != CurState) {
2059 if (Q->getRequiredState() > CurState)
2060 CurState = Q->getRequiredState();
2061 else
2062 LogFailure() << "symbol " << Sym
2063 << " has stale or misordered queries.\n";
2064 }
2065 }
2066
2067 // If there's a DefiningEDU then check that...
2068 // 1. The JD matches.
2069 // 2. The symbol is in the EDU's Symbols map.
2070 // 3. The symbol table entry is in the Emitted state.
2071 if (MII.DefiningEDU) {
2072
2073 EDUsToCheck.insert(MII.DefiningEDU.get());
2074
2075 if (MII.DefiningEDU->JD != JD.get()) {
2076 LogFailure() << "symbol " << Sym
2077 << " has DefiningEDU with incorrect JD"
2078 << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
2079 ? " (JD not currently in ExecutionSession"
2080 : "")
2081 << "\n";
2082 }
2083
2084 if (SymItr->second.getState() != SymbolState::Emitted) {
2085 LogFailure()
2086 << "symbol " << Sym
2087 << " has DefiningEDU, but is not in Emitted state.\n";
2088 }
2089 }
2090
2091 // Check that JDs for any DependantEDUs are also in the session --
2092 // that guarantees that we'll also visit them during this loop.
2093 for (auto &DepEDU : MII.DependantEDUs) {
2094 if (!llvm::is_contained(JDs, DepEDU->JD)) {
2095 LogFailure() << "symbol " << Sym << " has DependantEDU "
2096 << (void *)DepEDU << " with JD (" << DepEDU->JD
2097 << ") that isn't in ExecutionSession.\n";
2098 }
2099 }
2100 }
2101 }
2102 }
2103
2104 // Check EDUs.
2105 for (auto *EDU : EDUsToCheck) {
2106 assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
2107
2108 auto LogFailure = [&]() -> raw_fd_ostream & {
2109 AllOk = false;
2110 auto &Stream = errs();
2111 Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
2112 for (auto &[Sym, Flags] : EDU->Symbols)
2113 Stream << Sym << " ";
2114 Stream << "}, ";
2115 return Stream;
2116 };
2117
2118 if (EDU->Symbols.empty())
2119 LogFailure() << "no symbols defined.\n";
2120 else {
2121 for (auto &[Sym, Flags] : EDU->Symbols) {
2122 if (!Sym)
2123 LogFailure() << "null symbol defined.\n";
2124 else {
2125 if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
2126 LogFailure() << "symbol " << Sym
2127 << " isn't present in JD's symbol table.\n";
2128 }
2129 }
2130 }
2131 }
2132
2133 for (auto &[DepJD, Symbols] : EDU->Dependencies) {
2134 if (!llvm::is_contained(JDs, DepJD)) {
2135 LogFailure() << "dependant symbols listed for JD that isn't in "
2136 "ExecutionSession.\n";
2137 } else {
2138 for (auto &DepSym : Symbols) {
2139 if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
2140 LogFailure()
2141 << "dependant symbol " << DepSym
2142 << " does not appear in symbol table for dependant JD "
2143 << DepJD->getName() << ".\n";
2144 }
2145 }
2146 }
2147 }
2148 }
2149
2150 return AllOk;
2151 });
2152}
2153#endif // EXPENSIVE_CHECKS
2154
2155void ExecutionSession::dispatchOutstandingMUs() {
2156 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2157 while (true) {
2158 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2159 std::unique_ptr<MaterializationResponsibility>>>
2160 JMU;
2161
2162 {
2163 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2164 if (!OutstandingMUs.empty()) {
2165 JMU.emplace(args: std::move(OutstandingMUs.back()));
2166 OutstandingMUs.pop_back();
2167 }
2168 }
2169
2170 if (!JMU)
2171 break;
2172
2173 assert(JMU->first && "No MU?");
2174 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2175 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2176 args: std::move(JMU->second)));
2177 }
2178 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2179}
2180
2181Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2182 LLVM_DEBUG({
2183 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2184 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2185 });
2186 std::vector<ResourceManager *> CurrentResourceManagers;
2187
2188 JITDylib::RemoveTrackerResult R;
2189
2190 runSessionLocked(F: [&] {
2191 CurrentResourceManagers = ResourceManagers;
2192 RT.makeDefunct();
2193 R = RT.getJITDylib().IL_removeTracker(RT);
2194 });
2195
2196 // Release any defunct MaterializationUnits.
2197 R.DefunctMUs.clear();
2198
2199 Error Err = Error::success();
2200
2201 auto &JD = RT.getJITDylib();
2202 for (auto *L : reverse(C&: CurrentResourceManagers))
2203 Err = joinErrors(E1: std::move(Err),
2204 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2205
2206 for (auto &Q : R.QueriesToFail)
2207 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2208 Args&: R.FailedSymbols));
2209
2210 return Err;
2211}
2212
2213void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2214 ResourceTracker &SrcRT) {
2215 LLVM_DEBUG({
2216 dbgs() << "In " << SrcRT.getJITDylib().getName()
2217 << " transfering resources from tracker "
2218 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2219 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2220 });
2221
2222 // No-op transfers are allowed and do not invalidate the source.
2223 if (&DstRT == &SrcRT)
2224 return;
2225
2226 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2227 "Can't transfer resources between JITDylibs");
2228 runSessionLocked(F: [&]() {
2229 SrcRT.makeDefunct();
2230 auto &JD = DstRT.getJITDylib();
2231 JD.transferTracker(DstRT, SrcRT);
2232 for (auto *L : reverse(C&: ResourceManagers))
2233 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2234 SrcK: SrcRT.getKeyUnsafe());
2235 });
2236}
2237
2238void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2239 runSessionLocked(F: [&]() {
2240 LLVM_DEBUG({
2241 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2242 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2243 });
2244 if (!RT.isDefunct())
2245 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2246 SrcRT&: RT);
2247 });
2248}
2249
2250Error ExecutionSession::IL_updateCandidatesFor(
2251 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2252 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2253 return Candidates.forEachWithRemoval(
2254 Body: [&](const SymbolStringPtr &Name,
2255 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2256 /// Search for the symbol. If not found then continue without
2257 /// removal.
2258 auto SymI = JD.Symbols.find(Val: Name);
2259 if (SymI == JD.Symbols.end())
2260 return false;
2261
2262 // If this is a non-exported symbol and we're matching exported
2263 // symbols only then remove this symbol from the candidates list.
2264 //
2265 // If we're tracking non-candidates then add this to the non-candidate
2266 // list.
2267 if (!SymI->second.getFlags().isExported() &&
2268 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2269 if (NonCandidates)
2270 NonCandidates->add(Name, Flags: SymLookupFlags);
2271 return true;
2272 }
2273
2274 // If we match against a materialization-side-effects only symbol
2275 // then make sure it is weakly-referenced. Otherwise bail out with
2276 // an error.
2277 // FIXME: Use a "materialization-side-effects-only symbols must be
2278 // weakly referenced" specific error here to reduce confusion.
2279 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2280 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2281 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2282 Args: SymbolNameVector({Name}));
2283
2284 // If we matched against this symbol but it is in the error state
2285 // then bail out and treat it as a failure to materialize.
2286 if (SymI->second.getFlags().hasError()) {
2287 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2288 (*FailedSymbolsMap)[&JD] = {Name};
2289 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2290 Args: std::move(FailedSymbolsMap));
2291 }
2292
2293 // Otherwise this is a match. Remove it from the candidate set.
2294 return true;
2295 });
2296}
2297
2298void ExecutionSession::OL_resumeLookupAfterGeneration(
2299 InProgressLookupState &IPLS) {
2300
2301 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2302 "Should not be called for not-in-generator lookups");
2303 IPLS.GenState = InProgressLookupState::NotInGenerator;
2304
2305 LookupState LS;
2306
2307 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2308 IPLS.CurDefGeneratorStack.pop_back();
2309 std::lock_guard<std::mutex> Lock(DG->M);
2310
2311 // If there are no pending lookups then mark the generator as free and
2312 // return.
2313 if (DG->PendingLookups.empty()) {
2314 DG->InUse = false;
2315 return;
2316 }
2317
2318 // Otherwise resume the next lookup.
2319 LS = std::move(DG->PendingLookups.front());
2320 DG->PendingLookups.pop_front();
2321 }
2322
2323 if (LS.IPLS) {
2324 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2325 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2326 }
2327}
2328
2329void ExecutionSession::OL_applyQueryPhase1(
2330 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2331
2332 LLVM_DEBUG({
2333 dbgs() << "Entering OL_applyQueryPhase1:\n"
2334 << " Lookup kind: " << IPLS->K << "\n"
2335 << " Search order: " << IPLS->SearchOrder
2336 << ", Current index = " << IPLS->CurSearchOrderIndex
2337 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2338 << " Lookup set: " << IPLS->LookupSet << "\n"
2339 << " Definition generator candidates: "
2340 << IPLS->DefGeneratorCandidates << "\n"
2341 << " Definition generator non-candidates: "
2342 << IPLS->DefGeneratorNonCandidates << "\n";
2343 });
2344
2345 if (IPLS->GenState == InProgressLookupState::InGenerator)
2346 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2347
2348 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2349 "Lookup should not be in InGenerator state here");
2350
2351 // FIXME: We should attach the query as we go: This provides a result in a
2352 // single pass in the common case where all symbols have already reached the
2353 // required state. The query could be detached again in the 'fail' method on
2354 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2355
2356 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2357
2358 // If we've been handed an error or received one back from a generator then
2359 // fail the query. We don't need to unlink: At this stage the query hasn't
2360 // actually been lodged.
2361 if (Err)
2362 return IPLS->fail(Err: std::move(Err));
2363
2364 // Get the next JITDylib and lookup flags.
2365 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2366 auto &JD = *KV.first;
2367 auto JDLookupFlags = KV.second;
2368
2369 LLVM_DEBUG({
2370 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2371 << ") with lookup set " << IPLS->LookupSet << ":\n";
2372 });
2373
2374 // If we've just reached a new JITDylib then perform some setup.
2375 if (IPLS->NewJITDylib) {
2376 // Add any non-candidates from the last JITDylib (if any) back on to the
2377 // list of definition candidates for this JITDylib, reset definition
2378 // non-candidates to the empty set.
2379 SymbolLookupSet Tmp;
2380 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2381 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2382
2383 LLVM_DEBUG({
2384 dbgs() << " First time visiting " << JD.getName()
2385 << ", resetting candidate sets and building generator stack\n";
2386 });
2387
2388 // Build the definition generator stack for this JITDylib.
2389 runSessionLocked(F: [&] {
2390 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2391 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2392 R: reverse(C&: JD.DefGenerators));
2393 });
2394
2395 // Flag that we've done our initialization.
2396 IPLS->NewJITDylib = false;
2397 }
2398
2399 // Remove any generation candidates that are already defined (and match) in
2400 // this JITDylib.
2401 runSessionLocked(F: [&] {
2402 // Update the list of candidates (and non-candidates) for definition
2403 // generation.
2404 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2405 Err = IL_updateCandidatesFor(
2406 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2407 NonCandidates: JD.DefGenerators.empty() ? nullptr
2408 : &IPLS->DefGeneratorNonCandidates);
2409 LLVM_DEBUG({
2410 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2411 << "\n";
2412 });
2413
2414 // If this lookup was resumed after auto-suspension but all candidates
2415 // have already been generated (by some previous call to the generator)
2416 // treat the lookup as if it had completed generation.
2417 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2418 IPLS->DefGeneratorCandidates.empty())
2419 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2420 });
2421
2422 // If we encountered an error while filtering generation candidates then
2423 // bail out.
2424 if (Err)
2425 return IPLS->fail(Err: std::move(Err));
2426
2427 /// Apply any definition generators on the stack.
2428 LLVM_DEBUG({
2429 if (IPLS->CurDefGeneratorStack.empty())
2430 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2431 else if (IPLS->DefGeneratorCandidates.empty())
2432 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2433 else
2434 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2435 << " remaining generators for "
2436 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2437 });
2438 while (!IPLS->CurDefGeneratorStack.empty() &&
2439 !IPLS->DefGeneratorCandidates.empty()) {
2440 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2441
2442 if (!DG)
2443 return IPLS->fail(Err: make_error<StringError>(
2444 Args: "DefinitionGenerator removed while lookup in progress",
2445 Args: inconvertibleErrorCode()));
2446
2447 // At this point the lookup is in either the NotInGenerator state, or in
2448 // the ResumedForGenerator state.
2449 // If this lookup is in the NotInGenerator state then check whether the
2450 // generator is in use. If the generator is not in use then move the
2451 // lookup to the InGenerator state and continue. If the generator is
2452 // already in use then just add this lookup to the pending lookups list
2453 // and bail out.
2454 // If this lookup is in the ResumedForGenerator state then just move it
2455 // to InGenerator and continue.
2456 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2457 std::lock_guard<std::mutex> Lock(DG->M);
2458 if (DG->InUse) {
2459 DG->PendingLookups.push_back(x: std::move(IPLS));
2460 return;
2461 }
2462 DG->InUse = true;
2463 }
2464
2465 IPLS->GenState = InProgressLookupState::InGenerator;
2466
2467 auto K = IPLS->K;
2468 auto &LookupSet = IPLS->DefGeneratorCandidates;
2469
2470 // Run the generator. If the generator takes ownership of QA then this
2471 // will break the loop.
2472 {
2473 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2474 LookupState LS(std::move(IPLS));
2475 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2476 IPLS = std::move(LS.IPLS);
2477 }
2478
2479 // If the lookup returned then pop the generator stack and unblock the
2480 // next lookup on this generator (if any).
2481 if (IPLS)
2482 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2483
2484 // If there was an error then fail the query.
2485 if (Err) {
2486 LLVM_DEBUG({
2487 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2488 });
2489 assert(IPLS && "LS cannot be retained if error is returned");
2490 return IPLS->fail(Err: std::move(Err));
2491 }
2492
2493 // Otherwise if QA was captured then break the loop.
2494 if (!IPLS) {
2495 LLVM_DEBUG(
2496 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2497 return;
2498 }
2499
2500 // Otherwise if we're continuing around the loop then update candidates
2501 // for the next round.
2502 runSessionLocked(F: [&] {
2503 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2504 Err = IL_updateCandidatesFor(
2505 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2506 NonCandidates: JD.DefGenerators.empty() ? nullptr
2507 : &IPLS->DefGeneratorNonCandidates);
2508 });
2509
2510 // If updating candidates failed then fail the query.
2511 if (Err) {
2512 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2513 return IPLS->fail(Err: std::move(Err));
2514 }
2515 }
2516
2517 if (IPLS->DefGeneratorCandidates.empty() &&
2518 IPLS->DefGeneratorNonCandidates.empty()) {
2519 // Early out if there are no remaining symbols.
2520 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2521 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2522 break;
2523 } else {
2524 // If we get here then we've moved on to the next JITDylib with candidates
2525 // remaining.
2526 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2527 ++IPLS->CurSearchOrderIndex;
2528 IPLS->NewJITDylib = true;
2529 }
2530 }
2531
2532 // Remove any weakly referenced candidates that could not be found/generated.
2533 IPLS->DefGeneratorCandidates.remove_if(
2534 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2535 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2536 });
2537
2538 // If we get here then we've finished searching all JITDylibs.
2539 // If we matched all symbols then move to phase 2, otherwise fail the query
2540 // with a SymbolsNotFound error.
2541 if (IPLS->DefGeneratorCandidates.empty()) {
2542 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2543 IPLS->complete(IPLS: std::move(IPLS));
2544 } else {
2545 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2546 IPLS->fail(Err: make_error<SymbolsNotFound>(
2547 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2548 }
2549}
2550
2551void ExecutionSession::OL_completeLookup(
2552 std::unique_ptr<InProgressLookupState> IPLS,
2553 std::shared_ptr<AsynchronousSymbolQuery> Q,
2554 RegisterDependenciesFunction RegisterDependencies) {
2555
2556 LLVM_DEBUG({
2557 dbgs() << "Entering OL_completeLookup:\n"
2558 << " Lookup kind: " << IPLS->K << "\n"
2559 << " Search order: " << IPLS->SearchOrder
2560 << ", Current index = " << IPLS->CurSearchOrderIndex
2561 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2562 << " Lookup set: " << IPLS->LookupSet << "\n"
2563 << " Definition generator candidates: "
2564 << IPLS->DefGeneratorCandidates << "\n"
2565 << " Definition generator non-candidates: "
2566 << IPLS->DefGeneratorNonCandidates << "\n";
2567 });
2568
2569 bool QueryComplete = false;
2570 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2571
2572 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2573 for (auto &KV : IPLS->SearchOrder) {
2574 auto &JD = *KV.first;
2575 auto JDLookupFlags = KV.second;
2576 LLVM_DEBUG({
2577 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2578 << ") with lookup set " << IPLS->LookupSet << ":\n";
2579 });
2580
2581 auto Err = IPLS->LookupSet.forEachWithRemoval(
2582 Body: [&](const SymbolStringPtr &Name,
2583 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2584 LLVM_DEBUG({
2585 dbgs() << " Attempting to match \"" << Name << "\" ("
2586 << SymLookupFlags << ")... ";
2587 });
2588
2589 /// Search for the symbol. If not found then continue without
2590 /// removal.
2591 auto SymI = JD.Symbols.find(Val: Name);
2592 if (SymI == JD.Symbols.end()) {
2593 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2594 return false;
2595 }
2596
2597 // If this is a non-exported symbol and we're matching exported
2598 // symbols only then skip this symbol without removal.
2599 if (!SymI->second.getFlags().isExported() &&
2600 JDLookupFlags ==
2601 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2602 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2603 return false;
2604 }
2605
2606 // If we match against a materialization-side-effects only symbol
2607 // then make sure it is weakly-referenced. Otherwise bail out with
2608 // an error.
2609 // FIXME: Use a "materialization-side-effects-only symbols must be
2610 // weakly referenced" specific error here to reduce confusion.
2611 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2612 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2613 LLVM_DEBUG({
2614 dbgs() << "error: "
2615 "required, but symbol is has-side-effects-only\n";
2616 });
2617 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2618 Args: SymbolNameVector({Name}));
2619 }
2620
2621 // If we matched against this symbol but it is in the error state
2622 // then bail out and treat it as a failure to materialize.
2623 if (SymI->second.getFlags().hasError()) {
2624 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2625 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2626 (*FailedSymbolsMap)[&JD] = {Name};
2627 return make_error<FailedToMaterialize>(
2628 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2629 }
2630
2631 // Otherwise this is a match.
2632
2633 // If this symbol is already in the required state then notify the
2634 // query, remove the symbol and continue.
2635 if (SymI->second.getState() >= Q->getRequiredState()) {
2636 LLVM_DEBUG(dbgs()
2637 << "matched, symbol already in required state\n");
2638 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2639
2640 // If this symbol is in anything other than the Ready state then
2641 // we need to track the dependence.
2642 if (SymI->second.getState() != SymbolState::Ready)
2643 Q->addQueryDependence(JD, Name);
2644
2645 return true;
2646 }
2647
2648 // Otherwise this symbol does not yet meet the required state. Check
2649 // whether it has a materializer attached, and if so prepare to run
2650 // it.
2651 if (SymI->second.hasMaterializerAttached()) {
2652 assert(SymI->second.getAddress() == ExecutorAddr() &&
2653 "Symbol not resolved but already has address?");
2654 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2655 assert(UMII != JD.UnmaterializedInfos.end() &&
2656 "Lazy symbol should have UnmaterializedInfo");
2657
2658 auto UMI = UMII->second;
2659 assert(UMI->MU && "Materializer should not be null");
2660 assert(UMI->RT && "Tracker should not be null");
2661 LLVM_DEBUG({
2662 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2663 << " (" << UMI->MU->getName() << ")\n";
2664 });
2665
2666 // Move all symbols associated with this MaterializationUnit into
2667 // materializing state.
2668 for (auto &KV : UMI->MU->getSymbols()) {
2669 auto SymK = JD.Symbols.find(Val: KV.first);
2670 assert(SymK != JD.Symbols.end() &&
2671 "No entry for symbol covered by MaterializationUnit");
2672 SymK->second.setMaterializerAttached(false);
2673 SymK->second.setState(SymbolState::Materializing);
2674 JD.UnmaterializedInfos.erase(Val: KV.first);
2675 }
2676
2677 // Add MU to the list of MaterializationUnits to be materialized.
2678 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2679 } else
2680 LLVM_DEBUG(dbgs() << "matched, registering query");
2681
2682 // Add the query to the PendingQueries list and continue, deleting
2683 // the element from the lookup set.
2684 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2685 SymI->second.getState() != SymbolState::Ready &&
2686 "By this line the symbol should be materializing");
2687 auto &MI = JD.MaterializingInfos[Name];
2688 MI.addQuery(Q);
2689 Q->addQueryDependence(JD, Name);
2690
2691 return true;
2692 });
2693
2694 JD.shrinkMaterializationInfoMemory();
2695
2696 // Handle failure.
2697 if (Err) {
2698
2699 LLVM_DEBUG({
2700 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2701 });
2702
2703 // Detach the query.
2704 Q->detach();
2705
2706 // Replace the MUs.
2707 for (auto &KV : CollectedUMIs) {
2708 auto &JD = *KV.first;
2709 for (auto &UMI : KV.second)
2710 for (auto &KV2 : UMI->MU->getSymbols()) {
2711 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2712 "Unexpected materializer in map");
2713 auto SymI = JD.Symbols.find(Val: KV2.first);
2714 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2715 assert(SymI->second.getState() == SymbolState::Materializing &&
2716 "Can not replace symbol that is not materializing");
2717 assert(!SymI->second.hasMaterializerAttached() &&
2718 "MaterializerAttached flag should not be set");
2719 SymI->second.setMaterializerAttached(true);
2720 JD.UnmaterializedInfos[KV2.first] = UMI;
2721 }
2722 }
2723
2724 return Err;
2725 }
2726 }
2727
2728 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2729 IPLS->LookupSet.forEachWithRemoval(
2730 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2731 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2732 Q->dropSymbol(Name);
2733 return true;
2734 } else
2735 return false;
2736 });
2737
2738 if (!IPLS->LookupSet.empty()) {
2739 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2740 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2741 Args: IPLS->LookupSet.getSymbolNames());
2742 }
2743
2744 // Record whether the query completed.
2745 QueryComplete = Q->isComplete();
2746
2747 LLVM_DEBUG({
2748 dbgs() << "Query successfully "
2749 << (QueryComplete ? "completed" : "lodged") << "\n";
2750 });
2751
2752 // Move the collected MUs to the OutstandingMUs list.
2753 if (!CollectedUMIs.empty()) {
2754 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2755
2756 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2757 for (auto &KV : CollectedUMIs) {
2758 LLVM_DEBUG({
2759 auto &JD = *KV.first;
2760 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2761 << " MUs.\n";
2762 });
2763 for (auto &UMI : KV.second) {
2764 auto MR = createMaterializationResponsibility(
2765 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2766 InitSymbol: std::move(UMI->MU->InitSymbol));
2767 OutstandingMUs.push_back(
2768 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2769 }
2770 }
2771 } else
2772 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2773
2774 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2775 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2776 RegisterDependencies(Q->QueryRegistrations);
2777 } else
2778 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2779
2780 return Error::success();
2781 });
2782
2783 if (LodgingErr) {
2784 LLVM_DEBUG(dbgs() << "Failing query\n");
2785 Q->detach();
2786 Q->handleFailed(Err: std::move(LodgingErr));
2787 return;
2788 }
2789
2790 if (QueryComplete) {
2791 LLVM_DEBUG(dbgs() << "Completing query\n");
2792 Q->handleComplete(ES&: *this);
2793 }
2794
2795 dispatchOutstandingMUs();
2796}
2797
2798void ExecutionSession::OL_completeLookupFlags(
2799 std::unique_ptr<InProgressLookupState> IPLS,
2800 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2801
2802 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2803 LLVM_DEBUG({
2804 dbgs() << "Entering OL_completeLookupFlags:\n"
2805 << " Lookup kind: " << IPLS->K << "\n"
2806 << " Search order: " << IPLS->SearchOrder
2807 << ", Current index = " << IPLS->CurSearchOrderIndex
2808 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2809 << " Lookup set: " << IPLS->LookupSet << "\n"
2810 << " Definition generator candidates: "
2811 << IPLS->DefGeneratorCandidates << "\n"
2812 << " Definition generator non-candidates: "
2813 << IPLS->DefGeneratorNonCandidates << "\n";
2814 });
2815
2816 SymbolFlagsMap Result;
2817
2818 // Attempt to find flags for each symbol.
2819 for (auto &KV : IPLS->SearchOrder) {
2820 auto &JD = *KV.first;
2821 auto JDLookupFlags = KV.second;
2822 LLVM_DEBUG({
2823 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2824 << ") with lookup set " << IPLS->LookupSet << ":\n";
2825 });
2826
2827 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2828 SymbolLookupFlags SymLookupFlags) {
2829 LLVM_DEBUG({
2830 dbgs() << " Attempting to match \"" << Name << "\" ("
2831 << SymLookupFlags << ")... ";
2832 });
2833
2834 // Search for the symbol. If not found then continue without removing
2835 // from the lookup set.
2836 auto SymI = JD.Symbols.find(Val: Name);
2837 if (SymI == JD.Symbols.end()) {
2838 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2839 return false;
2840 }
2841
2842 // If this is a non-exported symbol then it doesn't match. Skip it.
2843 if (!SymI->second.getFlags().isExported() &&
2844 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2845 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2846 return false;
2847 }
2848
2849 LLVM_DEBUG({
2850 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2851 << "\n";
2852 });
2853 Result[Name] = SymI->second.getFlags();
2854 return true;
2855 });
2856 }
2857
2858 // Remove any weakly referenced symbols that haven't been resolved.
2859 IPLS->LookupSet.remove_if(
2860 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2861 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2862 });
2863
2864 if (!IPLS->LookupSet.empty()) {
2865 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2866 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2867 Args: IPLS->LookupSet.getSymbolNames());
2868 }
2869
2870 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2871 return Result;
2872 });
2873
2874 // Run the callback on the result.
2875 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2876 OnComplete(std::move(Result));
2877}
2878
2879void ExecutionSession::OL_destroyMaterializationResponsibility(
2880 MaterializationResponsibility &MR) {
2881
2882 assert(MR.SymbolFlags.empty() &&
2883 "All symbols should have been explicitly materialized or failed");
2884 MR.JD.unlinkMaterializationResponsibility(MR);
2885}
2886
2887SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2888 const MaterializationResponsibility &MR) {
2889 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2890}
2891
2892Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2893 const SymbolMap &Symbols) {
2894 LLVM_DEBUG({
2895 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2896 });
2897#ifndef NDEBUG
2898 for (auto &KV : Symbols) {
2899 auto I = MR.SymbolFlags.find(KV.first);
2900 assert(I != MR.SymbolFlags.end() &&
2901 "Resolving symbol outside this responsibility set");
2902 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2903 "Can't resolve materialization-side-effects-only symbol");
2904 if (I->second & JITSymbolFlags::Common) {
2905 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2906 assert((KV.second.getFlags() & WeakOrCommon) &&
2907 "Common symbols must be resolved as common or weak");
2908 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2909 (I->second & ~JITSymbolFlags::Common) &&
2910 "Resolving symbol with incorrect flags");
2911 } else
2912 assert(KV.second.getFlags() == I->second &&
2913 "Resolving symbol with incorrect flags");
2914 }
2915#endif
2916
2917 return MR.JD.resolve(MR, Resolved: Symbols);
2918}
2919
2920template <typename HandleNewDepFn>
2921void ExecutionSession::propagateExtraEmitDeps(
2922 std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
2923 HandleNewDepFn HandleNewDep) {
2924
2925 // Iterate to a fixed-point to propagate extra-emit dependencies through the
2926 // EDU graph.
2927 while (!Worklist.empty()) {
2928 auto &EDU = *Worklist.front();
2929 Worklist.pop_front();
2930
2931 assert(EDUInfos.count(&EDU) && "No info entry for EDU");
2932 auto &EDUInfo = EDUInfos[&EDU];
2933
2934 // Propagate new dependencies to users.
2935 for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
2936
2937 // UserEDUInfo only present if UserEDU has its own users.
2938 JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
2939 {
2940 auto UserEDUInfoItr = EDUInfos.find(Val: UserEDU);
2941 if (UserEDUInfoItr != EDUInfos.end())
2942 UserEDUInfo = &UserEDUInfoItr->second;
2943 }
2944
2945 for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
2946 auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
2947 DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
2948 for (auto Dep : Deps) {
2949 if (UserEDUDepsForJD.insert(V: Dep).second) {
2950 HandleNewDep(*UserEDU, *DepJD, Dep);
2951 if (UserEDUInfo) {
2952 if (!UserEDUNewDepsForJD) {
2953 // If UserEDU has no new deps then it's not in the worklist
2954 // yet, so add it.
2955 if (UserEDUInfo->NewDeps.empty())
2956 Worklist.push_back(x: UserEDU);
2957 UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
2958 }
2959 // Add (DepJD, Dep) to NewDeps.
2960 UserEDUNewDepsForJD->insert(V: Dep);
2961 }
2962 }
2963 }
2964 }
2965 }
2966
2967 EDUInfo.NewDeps.clear();
2968 }
2969}
2970
2971// Note: This method modifies the emitted set.
2972ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
2973 MaterializationResponsibility &MR,
2974 ArrayRef<SymbolDependenceGroup> EmittedDeps) {
2975
2976 auto &TargetJD = MR.getTargetJITDylib();
2977
2978 // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
2979 // Symbol -> EmissionDepUnit mappings.
2980 DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
2981 EDUInfos.reserve(NumEntries: EmittedDeps.size());
2982 DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
2983 for (auto &DG : EmittedDeps) {
2984 assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
2985
2986 // Skip empty EDUs.
2987 if (DG.Dependencies.empty())
2988 continue;
2989
2990 auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(args&: TargetJD);
2991 auto &EDUInfo = EDUInfos[TmpEDU.get()];
2992 EDUInfo.EDU = std::move(TmpEDU);
2993 for (const auto &Symbol : DG.Symbols) {
2994 NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
2995 assert(!EDUForSymbol.count(NonOwningSymbol) &&
2996 "Symbol should not appear in more than one SymbolDependenceGroup");
2997 assert(MR.getSymbols().count(Symbol) &&
2998 "Symbol in DepGroups not in the emitted set");
2999 auto NewlyEmittedItr = MR.getSymbols().find(Val: Symbol);
3000 EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
3001 EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
3002 }
3003 }
3004
3005 // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
3006 {
3007 DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
3008 for (auto &[Sym, Flags] : MR.getSymbols()) {
3009 if (!EDUForSymbol.count(Val: NonOwningSymbolStringPtr(Sym)))
3010 ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
3011 }
3012 if (!ResidualSymbolFlags.empty()) {
3013 auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(args&: TargetJD);
3014 ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
3015 auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
3016 ResidualEDUInfo.EDU = std::move(ResidualEDU);
3017
3018 // If the residual EDU is the only one then bail out early.
3019 if (EDUInfos.size() == 1)
3020 return EDUInfos;
3021
3022 // Otherwise add the residual EDU to the EDUForSymbol map.
3023 for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
3024 EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
3025 }
3026 }
3027
3028#ifndef NDEBUG
3029 assert(EDUForSymbol.size() == MR.getSymbols().size() &&
3030 "MR symbols not fully covered by EDUs?");
3031 for (auto &[Sym, Flags] : MR.getSymbols()) {
3032 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
3033 "Sym in MR not covered by EDU");
3034 }
3035#endif // NDEBUG
3036
3037 // 3. Use the DepGroups array to build a graph of dependencies between
3038 // EmissionDepUnits in this finalization. We want to remove these
3039 // intra-finalization uses, propagating dependencies on symbols outside
3040 // this finalization. Add EDUs to the worklist.
3041 for (auto &DG : EmittedDeps) {
3042
3043 // Skip SymbolDependenceGroups with no dependencies.
3044 if (DG.Dependencies.empty())
3045 continue;
3046
3047 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
3048 "No EDU for DG");
3049 auto &EDU =
3050 *EDUForSymbol.find(Val: NonOwningSymbolStringPtr(*DG.Symbols.begin()))
3051 ->second;
3052
3053 for (auto &[DepJD, Deps] : DG.Dependencies) {
3054 DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
3055
3056 assert(!Deps.empty() && "Dependence set for DepJD is empty");
3057
3058 if (DepJD != &TargetJD) {
3059 // DepJD is some other JITDylib.There can't be any intra-finalization
3060 // edges here, so just skip.
3061 for (auto &Dep : Deps)
3062 NewDepsForJD.insert(V: NonOwningSymbolStringPtr(Dep));
3063 } else {
3064 // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
3065 // skipping any and recording the intra-finalization use instead.
3066 for (auto &Dep : Deps) {
3067 NonOwningSymbolStringPtr NonOwningDep(Dep);
3068 auto I = EDUForSymbol.find(Val: NonOwningDep);
3069 if (I == EDUForSymbol.end()) {
3070 if (!MR.getSymbols().count(Val: Dep))
3071 NewDepsForJD.insert(V: NonOwningDep);
3072 continue;
3073 }
3074
3075 if (I->second != &EDU)
3076 EDUInfos[I->second].IntraEmitUsers.insert(V: &EDU);
3077 }
3078 }
3079
3080 if (!NewDepsForJD.empty())
3081 EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
3082 }
3083 }
3084
3085 // 4. Build the worklist.
3086 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3087 for (auto &[EDU, EDUInfo] : EDUInfos) {
3088 // If this EDU has extra-finalization dependencies and intra-finalization
3089 // users then add it to the worklist.
3090 if (!EDU->Dependencies.empty()) {
3091 auto I = EDUInfos.find(Val: EDU);
3092 if (I != EDUInfos.end()) {
3093 auto &EDUInfo = I->second;
3094 if (!EDUInfo.IntraEmitUsers.empty()) {
3095 EDUInfo.NewDeps = EDU->Dependencies;
3096 Worklist.push_back(x: EDU);
3097 }
3098 }
3099 }
3100 }
3101
3102 // 4. Propagate dependencies through the EDU graph.
3103 propagateExtraEmitDeps(
3104 Worklist, EDUInfos,
3105 HandleNewDep: [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
3106
3107 return EDUInfos;
3108}
3109
3110void ExecutionSession::IL_makeEDUReady(
3111 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3112 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3113
3114 // The symbols for this EDU are ready.
3115 auto &JD = *EDU->JD;
3116
3117 for (auto &[Sym, Flags] : EDU->Symbols) {
3118 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3119 "JD does not have an entry for Sym");
3120 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3121
3122 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3123 Entry.getState() == SymbolState::Materializing) ||
3124 Entry.getState() == SymbolState::Resolved ||
3125 Entry.getState() == SymbolState::Emitted) &&
3126 "Emitting from state other than Resolved");
3127
3128 Entry.setState(SymbolState::Ready);
3129
3130 auto MII = JD.MaterializingInfos.find(Val: SymbolStringPtr(Sym));
3131
3132 // Check for pending queries.
3133 if (MII == JD.MaterializingInfos.end())
3134 continue;
3135 auto &MI = MII->second;
3136
3137 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Ready)) {
3138 Q->notifySymbolMetRequiredState(Name: SymbolStringPtr(Sym), Sym: Entry.getSymbol());
3139 if (Q->isComplete())
3140 Queries.insert(x: Q);
3141 Q->removeQueryDependence(JD, Name: SymbolStringPtr(Sym));
3142 }
3143
3144 JD.MaterializingInfos.erase(I: MII);
3145 }
3146
3147 JD.shrinkMaterializationInfoMemory();
3148}
3149
3150void ExecutionSession::IL_makeEDUEmitted(
3151 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3152 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3153
3154 // The symbols for this EDU are emitted, but not ready.
3155 auto &JD = *EDU->JD;
3156
3157 for (auto &[Sym, Flags] : EDU->Symbols) {
3158 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3159 "JD does not have an entry for Sym");
3160 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3161
3162 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3163 Entry.getState() == SymbolState::Materializing) ||
3164 Entry.getState() == SymbolState::Resolved ||
3165 Entry.getState() == SymbolState::Emitted) &&
3166 "Emitting from state other than Resolved");
3167
3168 if (Entry.getState() == SymbolState::Emitted) {
3169 // This was already emitted, so we can skip the rest of this loop.
3170#ifndef NDEBUG
3171 for (auto &[Sym, Flags] : EDU->Symbols) {
3172 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3173 "JD does not have an entry for Sym");
3174 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3175 assert(Entry.getState() == SymbolState::Emitted &&
3176 "Symbols for EDU in inconsistent state");
3177 assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
3178 "Emitted symbol has no MI");
3179 auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3180 assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
3181 "Already-emitted symbol has waiting-on-emitted queries");
3182 }
3183#endif // NDEBUG
3184 break;
3185 }
3186
3187 Entry.setState(SymbolState::Emitted);
3188 auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3189 MI.DefiningEDU = EDU;
3190
3191 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Emitted)) {
3192 Q->notifySymbolMetRequiredState(Name: SymbolStringPtr(Sym), Sym: Entry.getSymbol());
3193 if (Q->isComplete())
3194 Queries.insert(x: Q);
3195 }
3196 }
3197
3198 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3199 for (auto &Dep : Deps)
3200 DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
3201 V: EDU.get());
3202 }
3203}
3204
3205/// Removes the given dependence from EDU. If EDU's dependence set becomes
3206/// empty then this function adds an entry for it to the EDUInfos map.
3207/// Returns true if a new EDUInfosMap entry is added.
3208bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
3209 JITDylib &DepJD,
3210 NonOwningSymbolStringPtr DepSym,
3211 EDUInfosMap &EDUInfos) {
3212 assert(EDU.Dependencies.count(&DepJD) &&
3213 "JD does not appear in Dependencies of DependantEDU");
3214 assert(EDU.Dependencies[&DepJD].count(DepSym) &&
3215 "Symbol does not appear in Dependencies of DependantEDU");
3216 auto &JDDeps = EDU.Dependencies[&DepJD];
3217 JDDeps.erase(V: DepSym);
3218 if (JDDeps.empty()) {
3219 EDU.Dependencies.erase(Val: &DepJD);
3220 if (EDU.Dependencies.empty()) {
3221 // If the dependencies set has become empty then EDU _may_ be ready
3222 // (we won't know for sure until we've propagated the extra-emit deps).
3223 // Create an EDUInfo for it (if it doesn't have one already) so that
3224 // it'll be visited after propagation.
3225 auto &DepEDUInfo = EDUInfos[&EDU];
3226 if (!DepEDUInfo.EDU) {
3227 assert(EDU.JD->Symbols.count(
3228 SymbolStringPtr(EDU.Symbols.begin()->first)) &&
3229 "Missing symbol entry for first symbol in EDU");
3230 auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
3231 Val: SymbolStringPtr(EDU.Symbols.begin()->first));
3232 assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
3233 "Missing MI for first symbol in DependantEDU");
3234 DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
3235 return true;
3236 }
3237 }
3238 }
3239 return false;
3240}
3241
3242Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
3243 JITDylib &ClosedJD) {
3244 SymbolNameSet FailedSymbols;
3245 for (auto &[Sym, Flags] : EDU.Symbols)
3246 FailedSymbols.insert(V: SymbolStringPtr(Sym));
3247 SymbolDependenceMap BadDeps;
3248 for (auto &Dep : EDU.Dependencies[&ClosedJD])
3249 BadDeps[&ClosedJD].insert(V: SymbolStringPtr(Dep));
3250 return make_error<UnsatisfiedSymbolDependencies>(
3251 Args: ClosedJD.getExecutionSession().getSymbolStringPool(), Args&: EDU.JD,
3252 Args: std::move(FailedSymbols), Args: std::move(BadDeps),
3253 Args: ClosedJD.getName() + " is closed");
3254}
3255
3256Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
3257 JITDylib &BadJD,
3258 SymbolNameSet BadDeps) {
3259 SymbolNameSet FailedSymbols;
3260 for (auto &[Sym, Flags] : EDU.Symbols)
3261 FailedSymbols.insert(V: SymbolStringPtr(Sym));
3262 SymbolDependenceMap BadDepsMap;
3263 BadDepsMap[&BadJD] = std::move(BadDeps);
3264 return make_error<UnsatisfiedSymbolDependencies>(
3265 Args: BadJD.getExecutionSession().getSymbolStringPool(), Args: &BadJD,
3266 Args: std::move(FailedSymbols), Args: std::move(BadDepsMap),
3267 Args: "dependencies removed or in error state");
3268}
3269
3270Expected<JITDylib::AsynchronousSymbolQuerySet>
3271ExecutionSession::IL_emit(MaterializationResponsibility &MR,
3272 EDUInfosMap EDUInfos) {
3273
3274 if (MR.RT->isDefunct())
3275 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
3276
3277 auto &TargetJD = MR.getTargetJITDylib();
3278 if (TargetJD.State != JITDylib::Open)
3279 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
3280 " is defunct",
3281 Args: inconvertibleErrorCode());
3282#ifdef EXPENSIVE_CHECKS
3283 verifySessionState("entering ExecutionSession::IL_emit");
3284#endif
3285
3286 // Walk all EDUs:
3287 // 1. Verifying that dependencies are available (not removed or in the error
3288 // state.
3289 // 2. Removing any dependencies that are already Ready.
3290 // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
3291 // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
3292 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3293 for (auto &[EDU, _] : EDUInfos)
3294 Worklist.push_back(x: EDU);
3295
3296 for (auto *EDU : Worklist) {
3297 auto *EDUInfo = &EDUInfos[EDU];
3298
3299 SmallVector<JITDylib *> DepJDsToRemove;
3300 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3301 if (DepJD->State != JITDylib::Open)
3302 return makeJDClosedError(EDU&: *EDU, ClosedJD&: *DepJD);
3303
3304 SymbolNameSet BadDeps;
3305 SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
3306 for (auto &Dep : Deps) {
3307 auto DepEntryItr = DepJD->Symbols.find(Val: SymbolStringPtr(Dep));
3308
3309 // If this dep has been removed or moved to the error state then add it
3310 // to the bad deps set. We aggregate these bad deps for more
3311 // comprehensive error messages.
3312 if (DepEntryItr == DepJD->Symbols.end() ||
3313 DepEntryItr->second.getFlags().hasError()) {
3314 BadDeps.insert(V: SymbolStringPtr(Dep));
3315 continue;
3316 }
3317
3318 // If this dep isn't emitted yet then just add it to the NewDeps set to
3319 // be propagated.
3320 auto &DepEntry = DepEntryItr->second;
3321 if (DepEntry.getState() < SymbolState::Emitted) {
3322 EDUInfo->NewDeps[DepJD].insert(V: Dep);
3323 continue;
3324 }
3325
3326 // This dep has been emitted, so add it to the list to be removed from
3327 // EDU.
3328 DepsToRemove.push_back(Elt: Dep);
3329
3330 // If Dep is Ready then there's nothing further to do.
3331 if (DepEntry.getState() == SymbolState::Ready) {
3332 assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3333 "Unexpected MaterializationInfo attached to ready symbol");
3334 continue;
3335 }
3336
3337 // If we get here then Dep is Emitted. We need to look up its defining
3338 // EDU and add this EDU to the defining EDU's list of users (this means
3339 // creating an EDUInfos entry if the defining EDU doesn't have one
3340 // already).
3341 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3342 "Expected MaterializationInfo for emitted dependency");
3343 auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
3344 assert(DepMI.DefiningEDU &&
3345 "Emitted symbol does not have a defining EDU");
3346 assert(DepMI.DependantEDUs.empty() &&
3347 "Already-emitted symbol has dependant EDUs?");
3348 auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
3349 if (!DepEDUInfo.EDU) {
3350 // No EDUInfo yet -- build initial entry, and reset the EDUInfo
3351 // pointer, which we will have invalidated.
3352 EDUInfo = &EDUInfos[EDU];
3353 DepEDUInfo.EDU = DepMI.DefiningEDU;
3354 for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
3355 if (DepDepJD == &TargetJD) {
3356 for (auto &DepDep : DepDeps)
3357 if (!MR.getSymbols().count(Val: SymbolStringPtr(DepDep)))
3358 DepEDUInfo.NewDeps[DepDepJD].insert(V: DepDep);
3359 } else
3360 DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
3361 }
3362 }
3363 DepEDUInfo.IntraEmitUsers.insert(V: EDU);
3364 }
3365
3366 // Some dependencies were removed or in an error state -- error out.
3367 if (!BadDeps.empty())
3368 return makeUnsatisfiedDepsError(EDU&: *EDU, BadJD&: *DepJD, BadDeps: std::move(BadDeps));
3369
3370 // Remove the emitted / ready deps from DepJD.
3371 for (auto &Dep : DepsToRemove)
3372 Deps.erase(V: Dep);
3373
3374 // If there are no further deps in DepJD then flag it for removal too.
3375 if (Deps.empty())
3376 DepJDsToRemove.push_back(Elt: DepJD);
3377 }
3378
3379 // Remove any JDs whose dependence sets have become empty.
3380 for (auto &DepJD : DepJDsToRemove) {
3381 assert(EDU->Dependencies.count(DepJD) &&
3382 "Trying to remove non-existent dep entries");
3383 EDU->Dependencies.erase(Val: DepJD);
3384 }
3385
3386 // Now look for users of this EDU.
3387 for (auto &[Sym, Flags] : EDU->Symbols) {
3388 assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
3389 "Sym not present in symbol table");
3390 assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
3391 SymbolState::Resolved ||
3392 TargetJD.Symbols[SymbolStringPtr(Sym)]
3393 .getFlags()
3394 .hasMaterializationSideEffectsOnly()) &&
3395 "Emitting symbol not in the resolved state");
3396 assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
3397 "Symbol is already in an error state");
3398
3399 auto MII = TargetJD.MaterializingInfos.find(Val: SymbolStringPtr(Sym));
3400 if (MII == TargetJD.MaterializingInfos.end() ||
3401 MII->second.DependantEDUs.empty())
3402 continue;
3403
3404 for (auto &DependantEDU : MII->second.DependantEDUs) {
3405 if (IL_removeEDUDependence(EDU&: *DependantEDU, DepJD&: TargetJD, DepSym: Sym, EDUInfos))
3406 EDUInfo = &EDUInfos[EDU];
3407 EDUInfo->IntraEmitUsers.insert(V: DependantEDU);
3408 }
3409 MII->second.DependantEDUs.clear();
3410 }
3411 }
3412
3413 Worklist.clear();
3414 for (auto &[EDU, EDUInfo] : EDUInfos) {
3415 if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
3416 if (EDUInfo.NewDeps.empty())
3417 EDUInfo.NewDeps = EDU->Dependencies;
3418 Worklist.push_back(x: EDU);
3419 }
3420 }
3421
3422 propagateExtraEmitDeps(
3423 Worklist, EDUInfos,
3424 HandleNewDep: [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
3425 NonOwningSymbolStringPtr Sym) {
3426 JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(V: &EDU);
3427 });
3428
3429 JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
3430
3431 // Extract completed queries and lodge not-yet-ready EDUs in the
3432 // session.
3433 for (auto &[EDU, EDUInfo] : EDUInfos) {
3434 if (EDU->Dependencies.empty())
3435 IL_makeEDUReady(EDU: std::move(EDUInfo.EDU), Queries&: CompletedQueries);
3436 else
3437 IL_makeEDUEmitted(EDU: std::move(EDUInfo.EDU), Queries&: CompletedQueries);
3438 }
3439
3440#ifdef EXPENSIVE_CHECKS
3441 verifySessionState("exiting ExecutionSession::IL_emit");
3442#endif
3443
3444 return std::move(CompletedQueries);
3445}
3446
3447Error ExecutionSession::OL_notifyEmitted(
3448 MaterializationResponsibility &MR,
3449 ArrayRef<SymbolDependenceGroup> DepGroups) {
3450 LLVM_DEBUG({
3451 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
3452 << "\n";
3453 if (!DepGroups.empty()) {
3454 dbgs() << " Initial dependencies:\n";
3455 for (auto &SDG : DepGroups) {
3456 dbgs() << " Symbols: " << SDG.Symbols
3457 << ", Dependencies: " << SDG.Dependencies << "\n";
3458 }
3459 }
3460 });
3461
3462#ifndef NDEBUG
3463 SymbolNameSet Visited;
3464 for (auto &DG : DepGroups) {
3465 for (auto &Sym : DG.Symbols) {
3466 assert(MR.SymbolFlags.count(Sym) &&
3467 "DG contains dependence for symbol outside this MR");
3468 assert(Visited.insert(Sym).second &&
3469 "DG contains duplicate entries for Name");
3470 }
3471 }
3472#endif // NDEBUG
3473
3474 auto EDUInfos = simplifyDepGroups(MR, EmittedDeps: DepGroups);
3475
3476 LLVM_DEBUG({
3477 dbgs() << " Simplified dependencies:\n";
3478 for (auto &[EDU, EDUInfo] : EDUInfos) {
3479 dbgs() << " Symbols: { ";
3480 for (auto &[Sym, Flags] : EDU->Symbols)
3481 dbgs() << Sym << " ";
3482 dbgs() << "}, Dependencies: { ";
3483 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3484 dbgs() << "(" << DepJD->getName() << ", { ";
3485 for (auto &Dep : Deps)
3486 dbgs() << Dep << " ";
3487 dbgs() << "}) ";
3488 }
3489 dbgs() << "}\n";
3490 }
3491 });
3492
3493 auto CompletedQueries =
3494 runSessionLocked(F: [&]() { return IL_emit(MR, EDUInfos); });
3495
3496 // On error bail out.
3497 if (!CompletedQueries)
3498 return CompletedQueries.takeError();
3499
3500 MR.SymbolFlags.clear();
3501
3502 // Otherwise notify all the completed queries.
3503 for (auto &Q : *CompletedQueries) {
3504 assert(Q->isComplete() && "Q is not complete");
3505 Q->handleComplete(ES&: *this);
3506 }
3507
3508 return Error::success();
3509}
3510
3511Error ExecutionSession::OL_defineMaterializing(
3512 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3513
3514 LLVM_DEBUG({
3515 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3516 << NewSymbolFlags << "\n";
3517 });
3518 if (auto AcceptedDefs =
3519 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3520 // Add all newly accepted symbols to this responsibility object.
3521 for (auto &KV : *AcceptedDefs)
3522 MR.SymbolFlags.insert(KV);
3523 return Error::success();
3524 } else
3525 return AcceptedDefs.takeError();
3526}
3527
3528std::pair<JITDylib::AsynchronousSymbolQuerySet,
3529 std::shared_ptr<SymbolDependenceMap>>
3530ExecutionSession::IL_failSymbols(JITDylib &JD,
3531 const SymbolNameVector &SymbolsToFail) {
3532
3533#ifdef EXPENSIVE_CHECKS
3534 verifySessionState("entering ExecutionSession::IL_failSymbols");
3535#endif
3536
3537 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3538 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3539 auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
3540 JITDylib::AsynchronousSymbolQueryList ToDetach;
3541 for (auto &Q : MI.pendingQueries()) {
3542 // Add the query to the list to be failed and detach it.
3543 FailedQueries.insert(x: Q);
3544 ToDetach.push_back(x: Q);
3545 }
3546 for (auto &Q : ToDetach)
3547 Q->detach();
3548 assert(!MI.hasQueriesPending() && "Queries still pending after detach");
3549 };
3550
3551 for (auto &Name : SymbolsToFail) {
3552 (*FailedSymbolsMap)[&JD].insert(V: Name);
3553
3554 // Look up the symbol to fail.
3555 auto SymI = JD.Symbols.find(Val: Name);
3556
3557 // FIXME: Revisit this. We should be able to assert sequencing between
3558 // ResourceTracker removal and symbol failure.
3559 //
3560 // It's possible that this symbol has already been removed, e.g. if a
3561 // materialization failure happens concurrently with a ResourceTracker or
3562 // JITDylib removal. In that case we can safely skip this symbol and
3563 // continue.
3564 if (SymI == JD.Symbols.end())
3565 continue;
3566 auto &Sym = SymI->second;
3567
3568 // If the symbol is already in the error state then we must have visited
3569 // it earlier.
3570 if (Sym.getFlags().hasError()) {
3571 assert(!JD.MaterializingInfos.count(Name) &&
3572 "Symbol in error state still has MaterializingInfo");
3573 continue;
3574 }
3575
3576 // Move the symbol into the error state.
3577 Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
3578
3579 // FIXME: Come up with a sane mapping of state to
3580 // presence-of-MaterializingInfo so that we can assert presence / absence
3581 // here, rather than testing it.
3582 auto MII = JD.MaterializingInfos.find(Val: Name);
3583 if (MII == JD.MaterializingInfos.end())
3584 continue;
3585
3586 auto &MI = MII->second;
3587
3588 // Collect queries to be failed for this MII.
3589 ExtractFailedQueries(MI);
3590
3591 if (MI.DefiningEDU) {
3592 // If there is a DefiningEDU for this symbol then remove this
3593 // symbol from it.
3594 assert(MI.DependantEDUs.empty() &&
3595 "Symbol with DefiningEDU should not have DependantEDUs");
3596 assert(Sym.getState() >= SymbolState::Emitted &&
3597 "Symbol has EDU, should have been emitted");
3598 assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
3599 "Symbol does not appear in its DefiningEDU");
3600 MI.DefiningEDU->Symbols.erase(Val: NonOwningSymbolStringPtr(Name));
3601
3602 // Remove this EDU from the dependants lists of its dependencies.
3603 for (auto &[DepJD, DepSyms] : MI.DefiningEDU->Dependencies) {
3604 for (auto DepSym : DepSyms) {
3605 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3606 "DepSym not in DepJD");
3607 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3608 "DepSym has not MaterializingInfo");
3609 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3610 assert(SymMI.DependantEDUs.count(MI.DefiningEDU.get()) &&
3611 "DefiningEDU missing from DependantEDUs list of dependency");
3612 SymMI.DependantEDUs.erase(V: MI.DefiningEDU.get());
3613 }
3614 }
3615
3616 MI.DefiningEDU = nullptr;
3617 } else {
3618 // Otherwise if there are any EDUs waiting on this symbol then move
3619 // those symbols to the error state too, and deregister them from the
3620 // symbols that they depend on.
3621 // Note: We use a copy of DependantEDUs here since we'll be removing
3622 // from the original set as we go.
3623 for (auto &DependantEDU : MI.DependantEDUs) {
3624
3625 // Remove DependantEDU from all of its users DependantEDUs lists.
3626 for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
3627 for (auto DepSym : DepSyms) {
3628 // Skip self-reference to avoid invalidating the MI.DependantEDUs
3629 // map. We'll clear this later.
3630 if (DepJD == &JD && DepSym == Name)
3631 continue;
3632 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3633 "DepSym not in DepJD?");
3634 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3635 "DependantEDU not registered with symbol it depends on");
3636 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3637 assert(SymMI.DependantEDUs.count(DependantEDU) &&
3638 "DependantEDU missing from DependantEDUs list");
3639 SymMI.DependantEDUs.erase(V: DependantEDU);
3640 }
3641 }
3642
3643 // Move any symbols defined by DependantEDU into the error state and
3644 // fail any queries waiting on them.
3645 auto &DepJD = *DependantEDU->JD;
3646 auto DepEDUSymbols = std::move(DependantEDU->Symbols);
3647 for (auto &[DepName, Flags] : DepEDUSymbols) {
3648 auto DepSymItr = DepJD.Symbols.find(Val: SymbolStringPtr(DepName));
3649 assert(DepSymItr != DepJD.Symbols.end() &&
3650 "Symbol not present in table");
3651 auto &DepSym = DepSymItr->second;
3652
3653 assert(DepSym.getState() >= SymbolState::Emitted &&
3654 "Symbol has EDU, should have been emitted");
3655 assert(!DepSym.getFlags().hasError() &&
3656 "Symbol is already in the error state?");
3657 DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
3658 (*FailedSymbolsMap)[&DepJD].insert(V: SymbolStringPtr(DepName));
3659
3660 // This symbol has a defining EDU so its MaterializingInfo object must
3661 // exist.
3662 auto DepMIItr =
3663 DepJD.MaterializingInfos.find(Val: SymbolStringPtr(DepName));
3664 assert(DepMIItr != DepJD.MaterializingInfos.end() &&
3665 "Symbol has defining EDU but not MaterializingInfo");
3666 auto &DepMI = DepMIItr->second;
3667 assert(DepMI.DefiningEDU.get() == DependantEDU &&
3668 "Bad EDU dependence edge");
3669 assert(DepMI.DependantEDUs.empty() &&
3670 "Symbol was emitted, should not have any DependantEDUs");
3671 ExtractFailedQueries(DepMI);
3672 DepJD.MaterializingInfos.erase(Val: SymbolStringPtr(DepName));
3673 }
3674
3675 DepJD.shrinkMaterializationInfoMemory();
3676 }
3677
3678 MI.DependantEDUs.clear();
3679 }
3680
3681 assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
3682 assert(MI.DependantEDUs.empty() &&
3683 "DependantEDUs should have been removed above");
3684 assert(!MI.hasQueriesPending() &&
3685 "Can not delete MaterializingInfo with queries pending");
3686 JD.MaterializingInfos.erase(Val: Name);
3687 }
3688
3689 JD.shrinkMaterializationInfoMemory();
3690
3691#ifdef EXPENSIVE_CHECKS
3692 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3693#endif
3694
3695 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3696}
3697
3698void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3699
3700 LLVM_DEBUG({
3701 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3702 << MR.SymbolFlags << "\n";
3703 });
3704
3705 if (MR.SymbolFlags.empty())
3706 return;
3707
3708 SymbolNameVector SymbolsToFail;
3709 for (auto &[Name, Flags] : MR.SymbolFlags)
3710 SymbolsToFail.push_back(x: Name);
3711 MR.SymbolFlags.clear();
3712
3713 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3714 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3715
3716 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3717 // If the tracker is defunct then there's nothing to do here.
3718 if (MR.RT->isDefunct())
3719 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3720 std::shared_ptr<SymbolDependenceMap>>();
3721 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3722 });
3723
3724 for (auto &Q : FailedQueries)
3725 Q->handleFailed(
3726 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3727}
3728
3729Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3730 std::unique_ptr<MaterializationUnit> MU) {
3731 for (auto &KV : MU->getSymbols()) {
3732 assert(MR.SymbolFlags.count(KV.first) &&
3733 "Replacing definition outside this responsibility set");
3734 MR.SymbolFlags.erase(Val: KV.first);
3735 }
3736
3737 if (MU->getInitializerSymbol() == MR.InitSymbol)
3738 MR.InitSymbol = nullptr;
3739
3740 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3741 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3742 << "\n";
3743 }););
3744
3745 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3746}
3747
3748Expected<std::unique_ptr<MaterializationResponsibility>>
3749ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3750 const SymbolNameSet &Symbols) {
3751
3752 SymbolStringPtr DelegatedInitSymbol;
3753 SymbolFlagsMap DelegatedFlags;
3754
3755 for (auto &Name : Symbols) {
3756 auto I = MR.SymbolFlags.find(Val: Name);
3757 assert(I != MR.SymbolFlags.end() &&
3758 "Symbol is not tracked by this MaterializationResponsibility "
3759 "instance");
3760
3761 DelegatedFlags[Name] = std::move(I->second);
3762 if (Name == MR.InitSymbol)
3763 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3764
3765 MR.SymbolFlags.erase(I);
3766 }
3767
3768 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3769 InitSymbol: std::move(DelegatedInitSymbol));
3770}
3771
3772#ifndef NDEBUG
3773void ExecutionSession::dumpDispatchInfo(Task &T) {
3774 runSessionLocked([&]() {
3775 dbgs() << "Dispatching: ";
3776 T.printDescription(dbgs());
3777 dbgs() << "\n";
3778 });
3779}
3780#endif // NDEBUG
3781
3782} // End namespace orc.
3783} // End namespace llvm.
3784