1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/Core.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
14#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15#include "llvm/ExecutionEngine/Orc/InstanceSymbolGeneratorSPS.h"
16#include "llvm/ExecutionEngine/Orc/Shared/Mangler.h"
17#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
18#include "llvm/Support/FormatVariadic.h"
19#include "llvm/Support/MSVCErrorWorkarounds.h"
20#include "llvm/Support/raw_ostream.h"
21
22#include <condition_variable>
23#include <future>
24#include <optional>
25
26#define DEBUG_TYPE "orc"
27
28namespace llvm {
29namespace orc {
30
31char ResourceTrackerDefunct::ID = 0;
32char JITDylibDefunct::ID = 0;
33char FailedToMaterialize::ID = 0;
34char SymbolsNotFound::ID = 0;
35char SymbolsCouldNotBeRemoved::ID = 0;
36char MissingSymbolDefinitions::ID = 0;
37char UnexpectedSymbolDefinitions::ID = 0;
38char UnsatisfiedSymbolDependencies::ID = 0;
39char MaterializationTask::ID = 0;
40char LookupTask::ID = 0;
41
42RegisterDependenciesFunction NoDependenciesToRegister =
43 RegisterDependenciesFunction();
44
45void MaterializationUnit::anchor() {}
46
47ResourceTracker::ResourceTracker(JITDylibSP JD) {
48 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
49 "JITDylib must be two byte aligned");
50 JD->Retain();
51 JDAndFlag.store(i: reinterpret_cast<uintptr_t>(JD.get()));
52}
53
54ResourceTracker::~ResourceTracker() {
55 getJITDylib().getExecutionSession().destroyResourceTracker(RT&: *this);
56 getJITDylib().Release();
57}
58
59Error ResourceTracker::remove() {
60 return getJITDylib().getExecutionSession().removeResourceTracker(RT&: *this);
61}
62
63void ResourceTracker::transferTo(ResourceTracker &DstRT) {
64 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, SrcRT&: *this);
65}
66
67void ResourceTracker::makeDefunct() {
68 uintptr_t Val = JDAndFlag.load();
69 Val |= 0x1U;
70 JDAndFlag.store(i: Val);
71}
72
73ResourceManager::~ResourceManager() = default;
74
75ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT)
76 : RT(std::move(RT)) {}
77
78std::error_code ResourceTrackerDefunct::convertToErrorCode() const {
79 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
80}
81
82void ResourceTrackerDefunct::log(raw_ostream &OS) const {
83 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
84}
85
86std::error_code JITDylibDefunct::convertToErrorCode() const {
87 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
88}
89
90void JITDylibDefunct::log(raw_ostream &OS) const {
91 OS << "JITDylib " << JD->getName() << " (" << (void *)JD.get()
92 << ") is defunct";
93}
94
95FailedToMaterialize::FailedToMaterialize(
96 std::shared_ptr<SymbolStringPool> SSP,
97 std::shared_ptr<SymbolDependenceMap> Symbols)
98 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
99 assert(this->SSP && "String pool cannot be null");
100 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
101
102 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
103 // don't have to manually retain/release.
104 for (auto &[JD, Syms] : *this->Symbols)
105 JD->Retain();
106}
107
108FailedToMaterialize::~FailedToMaterialize() {
109 for (auto &[JD, Syms] : *Symbols)
110 JD->Release();
111}
112
113std::error_code FailedToMaterialize::convertToErrorCode() const {
114 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
115}
116
117void FailedToMaterialize::log(raw_ostream &OS) const {
118 OS << "Failed to materialize symbols: " << *Symbols;
119}
120
121UnsatisfiedSymbolDependencies::UnsatisfiedSymbolDependencies(
122 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
123 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
124 std::string Explanation)
125 : SSP(std::move(SSP)), JD(std::move(JD)),
126 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
127 Explanation(std::move(Explanation)) {}
128
129std::error_code UnsatisfiedSymbolDependencies::convertToErrorCode() const {
130 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
131}
132
133void UnsatisfiedSymbolDependencies::log(raw_ostream &OS) const {
134 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
135 << ", due to unsatisfied dependencies " << BadDeps;
136 if (!Explanation.empty())
137 OS << " (" << Explanation << ")";
138}
139
140SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
141 SymbolNameSet Symbols)
142 : SSP(std::move(SSP)) {
143 llvm::append_range(C&: this->Symbols, R&: Symbols);
144 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
145}
146
147SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
148 SymbolNameVector Symbols)
149 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
150 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
151}
152
153std::error_code SymbolsNotFound::convertToErrorCode() const {
154 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
155}
156
157void SymbolsNotFound::log(raw_ostream &OS) const {
158 OS << "Symbols not found: " << Symbols;
159}
160
161SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(
162 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
163 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
164 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
165}
166
167std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const {
168 return orcError(ErrCode: OrcErrorCode::UnknownORCError);
169}
170
171void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const {
172 OS << "Symbols could not be removed: " << Symbols;
173}
174
175std::error_code MissingSymbolDefinitions::convertToErrorCode() const {
176 return orcError(ErrCode: OrcErrorCode::MissingSymbolDefinitions);
177}
178
179void MissingSymbolDefinitions::log(raw_ostream &OS) const {
180 OS << "Missing definitions in module " << ModuleName
181 << ": " << Symbols;
182}
183
184std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const {
185 return orcError(ErrCode: OrcErrorCode::UnexpectedSymbolDefinitions);
186}
187
188void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
189 OS << "Unexpected definitions in module " << ModuleName
190 << ": " << Symbols;
191}
192
193AsynchronousSymbolQuery::AsynchronousSymbolQuery(
194 const SymbolLookupSet &Symbols, SymbolState RequiredState,
195 SymbolsResolvedCallback NotifyComplete)
196 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
197 assert(RequiredState >= SymbolState::Resolved &&
198 "Cannot query for a symbols that have not reached the resolve state "
199 "yet");
200
201 OutstandingSymbolsCount = Symbols.size();
202
203 for (auto &[Name, Flags] : Symbols)
204 ResolvedSymbols[Name] = ExecutorSymbolDef();
205}
206
207void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
208 const SymbolStringPtr &Name, ExecutorSymbolDef Sym) {
209 auto I = ResolvedSymbols.find(Val: Name);
210 assert(I != ResolvedSymbols.end() &&
211 "Resolving symbol outside the requested set");
212 assert(I->second == ExecutorSymbolDef() &&
213 "Redundantly resolving symbol Name");
214
215 // If this is a materialization-side-effects-only symbol then drop it,
216 // otherwise update its map entry with its resolved address.
217 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
218 ResolvedSymbols.erase(I);
219 else
220 I->second = std::move(Sym);
221 --OutstandingSymbolsCount;
222}
223
224void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
225 assert(OutstandingSymbolsCount == 0 &&
226 "Symbols remain, handleComplete called prematurely");
227
228 class RunQueryCompleteTask : public Task {
229 public:
230 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
231 SymbolsResolvedCallback NotifyComplete)
232 : ResolvedSymbols(std::move(ResolvedSymbols)),
233 NotifyComplete(std::move(NotifyComplete)) {}
234 void printDescription(raw_ostream &OS) override {
235 OS << "Execute query complete callback for " << ResolvedSymbols;
236 }
237 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
238
239 private:
240 SymbolMap ResolvedSymbols;
241 SymbolsResolvedCallback NotifyComplete;
242 };
243
244 auto T = std::make_unique<RunQueryCompleteTask>(args: std::move(ResolvedSymbols),
245 args: std::move(NotifyComplete));
246 NotifyComplete = SymbolsResolvedCallback();
247 ES.dispatchTask(T: std::move(T));
248}
249
250void AsynchronousSymbolQuery::handleFailed(Error Err) {
251 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
252 OutstandingSymbolsCount == 0 &&
253 "Query should already have been abandoned");
254 NotifyComplete(std::move(Err));
255 NotifyComplete = SymbolsResolvedCallback();
256}
257
258void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
259 SymbolStringPtr Name) {
260 bool Added = QueryRegistrations[&JD].insert(V: std::move(Name)).second;
261 (void)Added;
262 assert(Added && "Duplicate dependence notification?");
263}
264
265void AsynchronousSymbolQuery::removeQueryDependence(
266 JITDylib &JD, const SymbolStringPtr &Name) {
267 auto QRI = QueryRegistrations.find(Val: &JD);
268 assert(QRI != QueryRegistrations.end() &&
269 "No dependencies registered for JD");
270 assert(QRI->second.count(Name) && "No dependency on Name in JD");
271 QRI->second.erase(V: Name);
272 if (QRI->second.empty())
273 QueryRegistrations.erase(I: QRI);
274}
275
276void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
277 auto I = ResolvedSymbols.find(Val: Name);
278 assert(I != ResolvedSymbols.end() &&
279 "Redundant removal of weakly-referenced symbol");
280 ResolvedSymbols.erase(I);
281 --OutstandingSymbolsCount;
282}
283
284void AsynchronousSymbolQuery::detach() {
285 ResolvedSymbols.clear();
286 OutstandingSymbolsCount = 0;
287 for (auto &[JD, Syms] : QueryRegistrations)
288 JD->detachQueryHelper(Q&: *this, QuerySymbols: Syms);
289 QueryRegistrations.clear();
290}
291
292ReExportsMaterializationUnit::ReExportsMaterializationUnit(
293 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
294 SymbolAliasMap Aliases)
295 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
296 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
297
298StringRef ReExportsMaterializationUnit::getName() const {
299 return "<Reexports>";
300}
301
302void ReExportsMaterializationUnit::materialize(
303 std::unique_ptr<MaterializationResponsibility> R) {
304
305 auto &ES = R->getTargetJITDylib().getExecutionSession();
306 JITDylib &TgtJD = R->getTargetJITDylib();
307 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
308
309 // Find the set of requested aliases and aliasees. Return any unrequested
310 // aliases back to the JITDylib so as to not prematurely materialize any
311 // aliasees.
312 auto RequestedSymbols = R->getRequestedSymbols();
313 SymbolAliasMap RequestedAliases;
314
315 for (auto &Name : RequestedSymbols) {
316 auto I = Aliases.find(Val: Name);
317 assert(I != Aliases.end() && "Symbol not found in aliases map?");
318 RequestedAliases[Name] = std::move(I->second);
319 Aliases.erase(I);
320 }
321
322 LLVM_DEBUG({
323 ES.runSessionLocked([&]() {
324 dbgs() << "materializing reexports: target = " << TgtJD.getName()
325 << ", source = " << SrcJD.getName() << " " << RequestedAliases
326 << "\n";
327 });
328 });
329
330 if (!Aliases.empty()) {
331 auto Err = SourceJD ? R->replace(MU: reexports(SourceJD&: *SourceJD, Aliases: std::move(Aliases),
332 SourceJDLookupFlags))
333 : R->replace(MU: symbolAliases(Aliases: std::move(Aliases)));
334
335 if (Err) {
336 // FIXME: Should this be reported / treated as failure to materialize?
337 // Or should this be treated as a sanctioned bailing-out?
338 ES.reportError(Err: std::move(Err));
339 R->failMaterialization();
340 return;
341 }
342 }
343
344 // The OnResolveInfo struct will hold the aliases and responsibility for each
345 // query in the list.
346 struct OnResolveInfo {
347 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
348 SymbolAliasMap Aliases)
349 : R(std::move(R)), Aliases(std::move(Aliases)) {}
350
351 std::unique_ptr<MaterializationResponsibility> R;
352 SymbolAliasMap Aliases;
353 std::vector<SymbolDependenceGroup> SDGs;
354 };
355
356 // Build a list of queries to issue. In each round we build a query for the
357 // largest set of aliases that we can resolve without encountering a chain of
358 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
359 // query would be waiting on a symbol that it itself had to resolve. Creating
360 // a new query for each link in such a chain eliminates the possibility of
361 // deadlock. In practice chains are likely to be rare, and this algorithm will
362 // usually result in a single query to issue.
363
364 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
365 QueryInfos;
366 while (!RequestedAliases.empty()) {
367 SymbolNameSet ResponsibilitySymbols;
368 SymbolLookupSet QuerySymbols;
369 SymbolAliasMap QueryAliases;
370
371 // Collect as many aliases as we can without including a chain.
372 for (auto &[Alias, AliasInfo] : RequestedAliases) {
373 // Chain detected. Skip this symbol for this round.
374 if (&SrcJD == &TgtJD && (QueryAliases.count(Val: AliasInfo.Aliasee) ||
375 RequestedAliases.count(Val: AliasInfo.Aliasee)))
376 continue;
377
378 ResponsibilitySymbols.insert(V: Alias);
379 QuerySymbols.add(Name: AliasInfo.Aliasee,
380 Flags: AliasInfo.AliasFlags.hasMaterializationSideEffectsOnly()
381 ? SymbolLookupFlags::WeaklyReferencedSymbol
382 : SymbolLookupFlags::RequiredSymbol);
383 QueryAliases[Alias] = std::move(AliasInfo);
384 }
385
386 // Remove the aliases collected this round from the RequestedAliases map.
387 for (auto &KV : QueryAliases)
388 RequestedAliases.erase(Val: KV.first);
389
390 assert(!QuerySymbols.empty() && "Alias cycle detected!");
391
392 auto NewR = R->delegate(Symbols: ResponsibilitySymbols);
393 if (!NewR) {
394 ES.reportError(Err: NewR.takeError());
395 R->failMaterialization();
396 return;
397 }
398
399 auto QueryInfo = std::make_shared<OnResolveInfo>(args: std::move(*NewR),
400 args: std::move(QueryAliases));
401 QueryInfos.push_back(
402 x: make_pair(x: std::move(QuerySymbols), y: std::move(QueryInfo)));
403 }
404
405 // Issue the queries.
406 while (!QueryInfos.empty()) {
407 auto QuerySymbols = std::move(QueryInfos.back().first);
408 auto QueryInfo = std::move(QueryInfos.back().second);
409
410 QueryInfos.pop_back();
411
412 auto RegisterDependencies = [QueryInfo,
413 &SrcJD](const SymbolDependenceMap &Deps) {
414 // If there were no materializing symbols, just bail out.
415 if (Deps.empty())
416 return;
417
418 // Otherwise the only deps should be on SrcJD.
419 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
420 "Unexpected dependencies for reexports");
421
422 auto &SrcJDDeps = Deps.find(Val: &SrcJD)->second;
423
424 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
425 if (SrcJDDeps.count(V: AliasInfo.Aliasee))
426 QueryInfo->SDGs.push_back(x: {.Symbols: {Alias}, .Dependencies: {{&SrcJD, {AliasInfo.Aliasee}}}});
427 };
428
429 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
430 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
431 if (Result) {
432 SymbolMap ResolutionMap;
433 for (auto &KV : QueryInfo->Aliases) {
434 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
435 Result->count(KV.second.Aliasee)) &&
436 "Result map missing entry?");
437 // Don't try to resolve materialization-side-effects-only symbols.
438 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
439 continue;
440
441 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
442 KV.second.AliasFlags};
443 }
444 if (auto Err = QueryInfo->R->notifyResolved(Symbols: ResolutionMap)) {
445 ES.reportError(Err: std::move(Err));
446 QueryInfo->R->failMaterialization();
447 return;
448 }
449 if (auto Err = QueryInfo->R->notifyEmitted(EmittedDeps: QueryInfo->SDGs)) {
450 ES.reportError(Err: std::move(Err));
451 QueryInfo->R->failMaterialization();
452 return;
453 }
454 } else {
455 ES.reportError(Err: Result.takeError());
456 QueryInfo->R->failMaterialization();
457 }
458 };
459
460 ES.lookup(K: LookupKind::Static,
461 SearchOrder: JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
462 Symbols: QuerySymbols, RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnComplete),
463 RegisterDependencies: std::move(RegisterDependencies));
464 }
465}
466
467void ReExportsMaterializationUnit::discard(const JITDylib &JD,
468 const SymbolStringPtr &Name) {
469 assert(Aliases.count(Name) &&
470 "Symbol not covered by this MaterializationUnit");
471 Aliases.erase(Val: Name);
472}
473
474MaterializationUnit::Interface
475ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
476 SymbolFlagsMap SymbolFlags;
477 for (auto &KV : Aliases)
478 SymbolFlags[KV.first] = KV.second.AliasFlags;
479
480 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
481}
482
483Expected<SymbolAliasMap>
484buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) {
485 auto Flags = SourceJD.getExecutionSession().lookupFlags(
486 K: LookupKind::Static, SearchOrder: {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}},
487 Symbols: SymbolLookupSet(std::move(Symbols)));
488
489 if (!Flags)
490 return Flags.takeError();
491
492 SymbolAliasMap Result;
493 for (auto &Name : Symbols) {
494 assert(Flags->count(Name) && "Missing entry in flags map");
495 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
496 }
497
498 return Result;
499}
500
501class InProgressLookupState {
502public:
503 // FIXME: Reduce the number of SymbolStringPtrs here. See
504 // https://github.com/llvm/llvm-project/issues/55576.
505
506 InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
507 SymbolLookupSet LookupSet, SymbolState RequiredState)
508 : K(K), SearchOrder(std::move(SearchOrder)),
509 LookupSet(std::move(LookupSet)), RequiredState(RequiredState) {
510 DefGeneratorCandidates = this->LookupSet;
511 }
512 virtual ~InProgressLookupState() = default;
513 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
514 virtual void fail(Error Err) = 0;
515
516 LookupKind K;
517 JITDylibSearchOrder SearchOrder;
518 SymbolLookupSet LookupSet;
519 SymbolState RequiredState;
520
521 size_t CurSearchOrderIndex = 0;
522 bool NewJITDylib = true;
523 SymbolLookupSet DefGeneratorCandidates;
524 SymbolLookupSet DefGeneratorNonCandidates;
525
526 enum {
527 NotInGenerator, // Not currently using a generator.
528 ResumedForGenerator, // Resumed after being auto-suspended before generator.
529 InGenerator // Currently using generator.
530 } GenState = NotInGenerator;
531 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
532};
533
534class InProgressLookupFlagsState : public InProgressLookupState {
535public:
536 InProgressLookupFlagsState(
537 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
538 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete)
539 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
540 SymbolState::NeverSearched),
541 OnComplete(std::move(OnComplete)) {}
542
543 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
544 auto &ES = SearchOrder.front().first->getExecutionSession();
545 ES.OL_completeLookupFlags(IPLS: std::move(IPLS), OnComplete: std::move(OnComplete));
546 }
547
548 void fail(Error Err) override { OnComplete(std::move(Err)); }
549
550private:
551 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete;
552};
553
554class InProgressFullLookupState : public InProgressLookupState {
555public:
556 InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder,
557 SymbolLookupSet LookupSet,
558 SymbolState RequiredState,
559 std::shared_ptr<AsynchronousSymbolQuery> Q,
560 RegisterDependenciesFunction RegisterDependencies)
561 : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet),
562 RequiredState),
563 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
564 }
565
566 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
567 auto &ES = SearchOrder.front().first->getExecutionSession();
568 ES.OL_completeLookup(IPLS: std::move(IPLS), Q: std::move(Q),
569 RegisterDependencies: std::move(RegisterDependencies));
570 }
571
572 void fail(Error Err) override {
573 Q->detach();
574 Q->handleFailed(Err: std::move(Err));
575 }
576
577private:
578 std::shared_ptr<AsynchronousSymbolQuery> Q;
579 RegisterDependenciesFunction RegisterDependencies;
580};
581
582ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
583 JITDylibLookupFlags SourceJDLookupFlags,
584 SymbolPredicate Allow)
585 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
586 Allow(std::move(Allow)) {}
587
588Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K,
589 JITDylib &JD,
590 JITDylibLookupFlags JDLookupFlags,
591 const SymbolLookupSet &LookupSet) {
592 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
593
594 // Use lookupFlags to find the subset of symbols that match our lookup.
595 auto Flags = JD.getExecutionSession().lookupFlags(
596 K, SearchOrder: {{&SourceJD, JDLookupFlags}}, Symbols: LookupSet);
597 if (!Flags)
598 return Flags.takeError();
599
600 // Create an alias map.
601 orc::SymbolAliasMap AliasMap;
602 for (auto &KV : *Flags)
603 if (!Allow || Allow(KV.first))
604 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
605
606 if (AliasMap.empty())
607 return Error::success();
608
609 // Define the re-exports.
610 return JD.define(MU: reexports(SourceJD, Aliases: AliasMap, SourceJDLookupFlags));
611}
612
613LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
614 : IPLS(std::move(IPLS)) {}
615
616void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(p: IPLS); }
617
618LookupState::LookupState() = default;
619LookupState::LookupState(LookupState &&) = default;
620LookupState &LookupState::operator=(LookupState &&) = default;
621LookupState::~LookupState() = default;
622
623void LookupState::continueLookup(Error Err) {
624 assert(IPLS && "Cannot call continueLookup on empty LookupState");
625 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
626 ES.OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: std::move(Err));
627}
628
629DefinitionGenerator::~DefinitionGenerator() {
630 std::deque<LookupState> LookupsToFail;
631 {
632 std::lock_guard<std::mutex> Lock(M);
633 std::swap(x&: PendingLookups, y&: LookupsToFail);
634 InUse = false;
635 }
636
637 for (auto &LS : LookupsToFail)
638 LS.continueLookup(Err: make_error<StringError>(
639 Args: "Query waiting on DefinitionGenerator that was destroyed",
640 Args: inconvertibleErrorCode()));
641}
642
643JITDylib::~JITDylib() {
644 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
645}
646
647Error JITDylib::clear() {
648 std::vector<ResourceTrackerSP> TrackersToRemove;
649 ES.runSessionLocked(F: [&]() {
650 assert(State != Closed && "JD is defunct");
651 for (auto &KV : TrackerSymbols)
652 TrackersToRemove.push_back(x: KV.first);
653 TrackersToRemove.push_back(x: getDefaultResourceTracker());
654 });
655
656 Error Err = Error::success();
657 for (auto &RT : TrackersToRemove)
658 Err = joinErrors(E1: std::move(Err), E2: RT->remove());
659 return Err;
660}
661
662ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
663 return ES.runSessionLocked(F: [this] {
664 assert(State != Closed && "JD is defunct");
665 if (!DefaultTracker)
666 DefaultTracker = new ResourceTracker(this);
667 return DefaultTracker;
668 });
669}
670
671ResourceTrackerSP JITDylib::createResourceTracker() {
672 return ES.runSessionLocked(F: [this] {
673 assert(State == Open && "JD is defunct");
674 ResourceTrackerSP RT = new ResourceTracker(this);
675 return RT;
676 });
677}
678
679void JITDylib::removeGenerator(DefinitionGenerator &G) {
680 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
681 // session lock (since it may have to send errors to pending queries).
682 std::shared_ptr<DefinitionGenerator> TmpDG;
683
684 ES.runSessionLocked(F: [&] {
685 assert(State == Open && "JD is defunct");
686 auto I = llvm::find_if(Range&: DefGenerators,
687 P: [&](const std::shared_ptr<DefinitionGenerator> &H) {
688 return H.get() == &G;
689 });
690 assert(I != DefGenerators.end() && "Generator not found");
691 TmpDG = std::move(*I);
692 DefGenerators.erase(position: I);
693 });
694}
695
696Expected<SymbolFlagsMap>
697JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
698 SymbolFlagsMap SymbolFlags) {
699
700 return ES.runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
701 if (FromMR.RT->isDefunct())
702 return make_error<ResourceTrackerDefunct>(Args&: FromMR.RT);
703
704 std::vector<NonOwningSymbolStringPtr> AddedSyms;
705 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
706
707 for (auto &[Name, Flags] : SymbolFlags) {
708 auto EntryItr = Symbols.find(Val: Name);
709
710 // If the entry already exists...
711 if (EntryItr != Symbols.end()) {
712
713 // If this is a strong definition then error out.
714 if (!Flags.isWeak()) {
715 // Remove any symbols already added.
716 for (auto &S : AddedSyms)
717 Symbols.erase(I: Symbols.find_as(Val: S));
718
719 // FIXME: Return all duplicates.
720 return make_error<DuplicateDefinition>(
721 Args: std::string(*Name), Args: "defineMaterializing operation");
722 }
723
724 // Otherwise just make a note to discard this symbol after the loop.
725 RejectedWeakDefs.push_back(x: NonOwningSymbolStringPtr(Name));
726 continue;
727 } else
728 EntryItr =
729 Symbols.insert(KV: std::make_pair(x&: Name, y: SymbolTableEntry(Flags))).first;
730
731 AddedSyms.push_back(x: NonOwningSymbolStringPtr(Name));
732 EntryItr->second.setState(SymbolState::Materializing);
733 }
734
735 // Remove any rejected weak definitions from the SymbolFlags map.
736 while (!RejectedWeakDefs.empty()) {
737 SymbolFlags.erase(I: SymbolFlags.find_as(Val: RejectedWeakDefs.back()));
738 RejectedWeakDefs.pop_back();
739 }
740
741 return SymbolFlags;
742 });
743}
744
745Error JITDylib::replace(MaterializationResponsibility &FromMR,
746 std::unique_ptr<MaterializationUnit> MU) {
747 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
748 std::unique_ptr<MaterializationUnit> MustRunMU;
749 std::unique_ptr<MaterializationResponsibility> MustRunMR;
750
751 auto Err =
752 ES.runSessionLocked(F: [&, this]() -> Error {
753 if (FromMR.RT->isDefunct())
754 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
755
756#ifndef NDEBUG
757 for (auto &KV : MU->getSymbols()) {
758 auto SymI = Symbols.find(KV.first);
759 assert(SymI != Symbols.end() && "Replacing unknown symbol");
760 assert(SymI->second.getState() == SymbolState::Materializing &&
761 "Can not replace a symbol that ha is not materializing");
762 assert(!SymI->second.hasMaterializerAttached() &&
763 "Symbol should not have materializer attached already");
764 assert(UnmaterializedInfos.count(KV.first) == 0 &&
765 "Symbol being replaced should have no UnmaterializedInfo");
766 }
767#endif // NDEBUG
768
769 // If the tracker is defunct we need to bail out immediately.
770
771 // If any symbol has pending queries against it then we need to
772 // materialize MU immediately.
773 for (auto &KV : MU->getSymbols()) {
774 auto MII = MaterializingInfos.find(Val: KV.first);
775 if (MII != MaterializingInfos.end()) {
776 if (MII->second.hasQueriesPending()) {
777 MustRunMR = ES.createMaterializationResponsibility(
778 RT&: *FromMR.RT, Symbols: std::move(MU->SymbolFlags),
779 InitSymbol: std::move(MU->InitSymbol));
780 MustRunMU = std::move(MU);
781 return Error::success();
782 }
783 }
784 }
785
786 // Otherwise, make MU responsible for all the symbols.
787 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU),
788 args: FromMR.RT.get());
789 for (auto &KV : UMI->MU->getSymbols()) {
790 auto SymI = Symbols.find(Val: KV.first);
791 assert(SymI->second.getState() == SymbolState::Materializing &&
792 "Can not replace a symbol that is not materializing");
793 assert(!SymI->second.hasMaterializerAttached() &&
794 "Can not replace a symbol that has a materializer attached");
795 assert(UnmaterializedInfos.count(KV.first) == 0 &&
796 "Unexpected materializer entry in map");
797 SymI->second.setAddress(SymI->second.getAddress());
798 SymI->second.setMaterializerAttached(true);
799
800 auto &UMIEntry = UnmaterializedInfos[KV.first];
801 assert((!UMIEntry || !UMIEntry->MU) &&
802 "Replacing symbol with materializer still attached");
803 UMIEntry = UMI;
804 }
805
806 return Error::success();
807 });
808
809 if (Err)
810 return Err;
811
812 if (MustRunMU) {
813 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
814 ES.dispatchTask(T: std::make_unique<MaterializationTask>(
815 args: std::move(MustRunMU), args: std::move(MustRunMR)));
816 } else {
817 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
818 }
819
820 return Error::success();
821}
822
823Expected<std::unique_ptr<MaterializationResponsibility>>
824JITDylib::delegate(MaterializationResponsibility &FromMR,
825 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
826
827 return ES.runSessionLocked(
828 F: [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
829 if (FromMR.RT->isDefunct())
830 return make_error<ResourceTrackerDefunct>(Args: std::move(FromMR.RT));
831
832 return ES.createMaterializationResponsibility(
833 RT&: *FromMR.RT, Symbols: std::move(SymbolFlags), InitSymbol: std::move(InitSymbol));
834 });
835}
836
837SymbolNameSet
838JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
839 return ES.runSessionLocked(F: [&]() {
840 SymbolNameSet RequestedSymbols;
841
842 for (auto &KV : SymbolFlags) {
843 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
844 assert(Symbols.find(KV.first)->second.getState() !=
845 SymbolState::NeverSearched &&
846 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
847 "getRequestedSymbols can only be called for symbols that have "
848 "started materializing");
849 auto I = MaterializingInfos.find(Val: KV.first);
850 if (I == MaterializingInfos.end())
851 continue;
852
853 if (I->second.hasQueriesPending())
854 RequestedSymbols.insert(V: KV.first);
855 }
856
857 return RequestedSymbols;
858 });
859}
860
861Error JITDylib::resolve(MaterializationResponsibility &MR,
862 const SymbolMap &Resolved) {
863 AsynchronousSymbolQuerySet CompletedQueries;
864
865 if (auto Err = ES.runSessionLocked(F: [&, this]() -> Error {
866 if (MR.RT->isDefunct())
867 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
868
869 if (State != Open)
870 return make_error<StringError>(Args: "JITDylib " + getName() +
871 " is defunct",
872 Args: inconvertibleErrorCode());
873
874 struct WorklistEntry {
875 SymbolTable::iterator SymI;
876 ExecutorSymbolDef ResolvedSym;
877 };
878
879 SymbolNameSet SymbolsInErrorState;
880 std::vector<WorklistEntry> Worklist;
881 Worklist.reserve(n: Resolved.size());
882
883 // Build worklist and check for any symbols in the error state.
884 for (const auto &KV : Resolved) {
885
886 assert(!KV.second.getFlags().hasError() &&
887 "Resolution result can not have error flag set");
888
889 auto SymI = Symbols.find(Val: KV.first);
890
891 assert(SymI != Symbols.end() && "Symbol not found");
892 assert(!SymI->second.hasMaterializerAttached() &&
893 "Resolving symbol with materializer attached?");
894 assert(SymI->second.getState() == SymbolState::Materializing &&
895 "Symbol should be materializing");
896 assert(SymI->second.getAddress() == ExecutorAddr() &&
897 "Symbol has already been resolved");
898
899 if (SymI->second.getFlags().hasError())
900 SymbolsInErrorState.insert(V: KV.first);
901 else {
902 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
903 [[maybe_unused]] auto WeakOrCommon =
904 JITSymbolFlags::Weak | JITSymbolFlags::Common;
905 assert((KV.second.getFlags() & WeakOrCommon) &&
906 "Common symbols must be resolved as common or weak");
907 assert((KV.second.getFlags() & ~WeakOrCommon) ==
908 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
909 "Resolving symbol with incorrect flags");
910
911 } else
912 assert(KV.second.getFlags() == SymI->second.getFlags() &&
913 "Resolved flags should match the declared flags");
914
915 Worklist.push_back(
916 x: {.SymI: SymI, .ResolvedSym: {KV.second.getAddress(), SymI->second.getFlags()}});
917 }
918 }
919
920 // If any symbols were in the error state then bail out.
921 if (!SymbolsInErrorState.empty()) {
922 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
923 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
924 return make_error<FailedToMaterialize>(
925 Args: getExecutionSession().getSymbolStringPool(),
926 Args: std::move(FailedSymbolsDepMap));
927 }
928
929 while (!Worklist.empty()) {
930 auto SymI = Worklist.back().SymI;
931 auto ResolvedSym = Worklist.back().ResolvedSym;
932 Worklist.pop_back();
933
934 auto &Name = SymI->first;
935
936 // Resolved symbols can not be weak: discard the weak flag.
937 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
938 SymI->second.setAddress(ResolvedSym.getAddress());
939 SymI->second.setFlags(ResolvedFlags);
940 SymI->second.setState(SymbolState::Resolved);
941
942 auto MII = MaterializingInfos.find(Val: Name);
943 if (MII == MaterializingInfos.end())
944 continue;
945
946 auto &MI = MII->second;
947 for (auto &Q : MI.takeQueriesMeeting(RequiredState: SymbolState::Resolved)) {
948 Q->notifySymbolMetRequiredState(Name, Sym: ResolvedSym);
949 if (Q->isComplete())
950 CompletedQueries.insert(x: std::move(Q));
951 }
952 }
953
954 return Error::success();
955 }))
956 return Err;
957
958 // Otherwise notify all the completed queries.
959 for (auto &Q : CompletedQueries) {
960 assert(Q->isComplete() && "Q not completed");
961 Q->handleComplete(ES);
962 }
963
964 return Error::success();
965}
966
967void JITDylib::unlinkMaterializationResponsibility(
968 MaterializationResponsibility &MR) {
969 ES.runSessionLocked(F: [&]() {
970 auto I = TrackerMRs.find(Val: MR.RT.get());
971 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
972 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
973 I->second.erase(V: &MR);
974 if (I->second.empty())
975 TrackerMRs.erase(Val: MR.RT.get());
976 });
977}
978
979void JITDylib::shrinkMaterializationInfoMemory() {
980 // DenseMap::erase never shrinks its storage; use clear to heuristically free
981 // memory since we may have long-lived JDs after linking is done.
982
983 if (UnmaterializedInfos.empty())
984 UnmaterializedInfos.clear();
985
986 if (MaterializingInfos.empty())
987 MaterializingInfos.clear();
988}
989
990void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
991 bool LinkAgainstThisJITDylibFirst) {
992 ES.runSessionLocked(F: [&]() {
993 assert(State == Open && "JD is defunct");
994 if (LinkAgainstThisJITDylibFirst) {
995 LinkOrder.clear();
996 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
997 LinkOrder.push_back(
998 x: std::make_pair(x: this, y: JITDylibLookupFlags::MatchAllSymbols));
999 llvm::append_range(C&: LinkOrder, R&: NewLinkOrder);
1000 } else
1001 LinkOrder = std::move(NewLinkOrder);
1002 });
1003}
1004
1005void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1006 ES.runSessionLocked(F: [&]() {
1007 for (auto &KV : NewLinks) {
1008 // Skip elements of NewLinks that are already in the link order.
1009 if (llvm::is_contained(Range&: LinkOrder, Element: KV))
1010 continue;
1011
1012 LinkOrder.push_back(x: std::move(KV));
1013 }
1014 });
1015}
1016
1017void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1018 ES.runSessionLocked(F: [&]() { LinkOrder.push_back(x: {&JD, JDLookupFlags}); });
1019}
1020
1021void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1022 JITDylibLookupFlags JDLookupFlags) {
1023 ES.runSessionLocked(F: [&]() {
1024 assert(State == Open && "JD is defunct");
1025 for (auto &KV : LinkOrder)
1026 if (KV.first == &OldJD) {
1027 KV = {&NewJD, JDLookupFlags};
1028 break;
1029 }
1030 });
1031}
1032
1033void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1034 ES.runSessionLocked(F: [&]() {
1035 assert(State == Open && "JD is defunct");
1036 auto I = llvm::find_if(Range&: LinkOrder,
1037 P: [&](const JITDylibSearchOrder::value_type &KV) {
1038 return KV.first == &JD;
1039 });
1040 if (I != LinkOrder.end())
1041 LinkOrder.erase(position: I);
1042 });
1043}
1044
1045Error JITDylib::remove(const SymbolNameSet &Names) {
1046 return ES.runSessionLocked(F: [&]() -> Error {
1047 assert(State == Open && "JD is defunct");
1048 SmallVector<SymbolStringPtr, 0> SymbolsToRemove;
1049 SymbolNameSet Missing;
1050 SymbolNameSet Materializing;
1051
1052 for (auto &Name : Names) {
1053 auto I = Symbols.find(Val: Name);
1054
1055 // Note symbol missing.
1056 if (I == Symbols.end()) {
1057 Missing.insert(V: Name);
1058 continue;
1059 }
1060
1061 // Note symbol materializing.
1062 if (I->second.getState() != SymbolState::NeverSearched &&
1063 I->second.getState() != SymbolState::Ready) {
1064 Materializing.insert(V: Name);
1065 continue;
1066 }
1067
1068 SymbolsToRemove.push_back(Elt: Name);
1069 }
1070
1071 // If any of the symbols are not defined, return an error.
1072 if (!Missing.empty())
1073 return make_error<SymbolsNotFound>(Args: ES.getSymbolStringPool(),
1074 Args: std::move(Missing));
1075
1076 // If any of the symbols are currently materializing, return an error.
1077 if (!Materializing.empty())
1078 return make_error<SymbolsCouldNotBeRemoved>(Args: ES.getSymbolStringPool(),
1079 Args: std::move(Materializing));
1080
1081 // Remove the symbols. Erase by key rather than holding iterators across the
1082 // loop: a prior erase invalidates other stored iterators under
1083 // backward-shift deletion.
1084 for (const SymbolStringPtr &Name : SymbolsToRemove) {
1085 // If there is a materializer attached, call discard.
1086 auto UMII = UnmaterializedInfos.find(Val: Name);
1087 if (UMII != UnmaterializedInfos.end()) {
1088 UMII->second->MU->doDiscard(JD: *this, Name: UMII->first);
1089 UnmaterializedInfos.erase(I: UMII);
1090 }
1091
1092 Symbols.erase(Val: Name);
1093 }
1094
1095 shrinkMaterializationInfoMemory();
1096
1097 return Error::success();
1098 });
1099}
1100
1101void JITDylib::dump(raw_ostream &OS) {
1102 ES.runSessionLocked(F: [&, this]() {
1103 OS << "JITDylib \"" << getName() << "\" (ES: "
1104 << format(Fmt: "0x%016" PRIx64, Vals: reinterpret_cast<uintptr_t>(&ES))
1105 << ", State = ";
1106 switch (State) {
1107 case Open:
1108 OS << "Open";
1109 break;
1110 case Closing:
1111 OS << "Closing";
1112 break;
1113 case Closed:
1114 OS << "Closed";
1115 break;
1116 }
1117 OS << ")\n";
1118 if (State == Closed)
1119 return;
1120 OS << "Link order: " << LinkOrder << "\n"
1121 << "Symbol table:\n";
1122
1123 // Sort symbols so we get a deterministic order and can check them in tests.
1124 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1125 for (auto &KV : Symbols)
1126 SymbolsSorted.emplace_back(args&: KV.first, args: &KV.second);
1127 std::sort(first: SymbolsSorted.begin(), last: SymbolsSorted.end(),
1128 comp: [](const auto &L, const auto &R) { return *L.first < *R.first; });
1129
1130 for (auto &KV : SymbolsSorted) {
1131 OS << " \"" << *KV.first << "\": ";
1132 if (auto Addr = KV.second->getAddress())
1133 OS << Addr;
1134 else
1135 OS << "<not resolved> ";
1136
1137 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1138
1139 if (KV.second->hasMaterializerAttached()) {
1140 OS << " (Materializer ";
1141 auto I = UnmaterializedInfos.find(Val: KV.first);
1142 assert(I != UnmaterializedInfos.end() &&
1143 "Lazy symbol should have UnmaterializedInfo");
1144 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1145 } else
1146 OS << "\n";
1147 }
1148
1149 if (!MaterializingInfos.empty())
1150 OS << " MaterializingInfos entries:\n";
1151 for (auto &KV : MaterializingInfos) {
1152 OS << " \"" << *KV.first << "\":\n"
1153 << " " << KV.second.pendingQueries().size()
1154 << " pending queries: { ";
1155 for (const auto &Q : KV.second.pendingQueries())
1156 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1157 OS << "}\n";
1158 }
1159 });
1160}
1161
1162void JITDylib::MaterializingInfo::addQuery(
1163 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1164
1165 auto I = llvm::lower_bound(
1166 Range: llvm::reverse(C&: PendingQueries), Value: Q->getRequiredState(),
1167 C: [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1168 return V->getRequiredState() <= S;
1169 });
1170 PendingQueries.insert(position: I.base(), x: std::move(Q));
1171}
1172
1173void JITDylib::MaterializingInfo::removeQuery(
1174 const AsynchronousSymbolQuery &Q) {
1175 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1176 auto I = llvm::find_if(
1177 Range&: PendingQueries, P: [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1178 return V.get() == &Q;
1179 });
1180 if (I != PendingQueries.end())
1181 PendingQueries.erase(position: I);
1182}
1183
1184JITDylib::AsynchronousSymbolQueryList
1185JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1186 AsynchronousSymbolQueryList Result;
1187 while (!PendingQueries.empty()) {
1188 if (PendingQueries.back()->getRequiredState() > RequiredState)
1189 break;
1190
1191 Result.push_back(x: std::move(PendingQueries.back()));
1192 PendingQueries.pop_back();
1193 }
1194
1195 return Result;
1196}
1197
1198JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1199 : JITLinkDylib(std::move(Name)), ES(ES) {
1200 LinkOrder.push_back(x: {this, JITDylibLookupFlags::MatchAllSymbols});
1201}
1202
1203JITDylib::RemoveTrackerResult JITDylib::IL_removeTracker(ResourceTracker &RT) {
1204 // Note: Should be called under the session lock.
1205 assert(State != Closed && "JD is defunct");
1206
1207 SymbolNameVector SymbolsToRemove;
1208 SymbolNameVector SymbolsToFail;
1209
1210 if (&RT == DefaultTracker.get()) {
1211 SymbolNameSet TrackedSymbols;
1212 for (auto &KV : TrackerSymbols)
1213 TrackedSymbols.insert_range(R&: KV.second);
1214
1215 for (auto &KV : Symbols) {
1216 auto &Sym = KV.first;
1217 if (!TrackedSymbols.count(V: Sym))
1218 SymbolsToRemove.push_back(x: Sym);
1219 }
1220
1221 DefaultTracker.reset();
1222 } else {
1223 /// Check for a non-default tracker.
1224 auto I = TrackerSymbols.find(Val: &RT);
1225 if (I != TrackerSymbols.end()) {
1226 SymbolsToRemove = std::move(I->second);
1227 TrackerSymbols.erase(I);
1228 }
1229 // ... if not found this tracker was already defunct. Nothing to do.
1230 }
1231
1232 for (auto &Sym : SymbolsToRemove) {
1233 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1234
1235 // If there is a MaterializingInfo then collect any queries to fail.
1236 auto MII = MaterializingInfos.find(Val: Sym);
1237 if (MII != MaterializingInfos.end())
1238 SymbolsToFail.push_back(x: Sym);
1239 }
1240
1241 auto [QueriesToFail, FailedSymbols] =
1242 ES.IL_failSymbols(JD&: *this, SymbolsToFail: std::move(SymbolsToFail));
1243
1244 std::vector<std::unique_ptr<MaterializationUnit>> DefunctMUs;
1245
1246 // Removed symbols should be taken out of the table altogether.
1247 for (auto &Sym : SymbolsToRemove) {
1248 auto I = Symbols.find(Val: Sym);
1249 assert(I != Symbols.end() && "Symbol not present in table");
1250
1251 // Remove Materializer if present.
1252 if (I->second.hasMaterializerAttached()) {
1253 // FIXME: Should this discard the symbols?
1254 auto J = UnmaterializedInfos.find(Val: Sym);
1255 assert(J != UnmaterializedInfos.end() &&
1256 "Symbol table indicates MU present, but no UMI record");
1257 if (J->second->MU)
1258 DefunctMUs.push_back(x: std::move(J->second->MU));
1259 UnmaterializedInfos.erase(I: J);
1260 } else {
1261 assert(!UnmaterializedInfos.count(Sym) &&
1262 "Symbol has materializer attached");
1263 }
1264
1265 Symbols.erase(I);
1266 }
1267
1268 shrinkMaterializationInfoMemory();
1269
1270 return {.QueriesToFail: std::move(QueriesToFail), .FailedSymbols: std::move(FailedSymbols),
1271 .DefunctMUs: std::move(DefunctMUs)};
1272}
1273
1274void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1275 assert(State != Closed && "JD is defunct");
1276 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1277 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1278 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1279
1280 // Update trackers for any not-yet materialized units.
1281 for (auto &KV : UnmaterializedInfos) {
1282 if (KV.second->RT == &SrcRT)
1283 KV.second->RT = &DstRT;
1284 }
1285
1286 // Update trackers for any active materialization responsibilities.
1287 {
1288 auto I = TrackerMRs.find(Val: &SrcRT);
1289 if (I != TrackerMRs.end()) {
1290 auto &SrcMRs = I->second;
1291 auto &DstMRs = TrackerMRs[&DstRT];
1292 for (auto *MR : SrcMRs)
1293 MR->RT = &DstRT;
1294 if (DstMRs.empty())
1295 DstMRs = std::move(SrcMRs);
1296 else
1297 DstMRs.insert_range(R&: SrcMRs);
1298 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1299 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1300 TrackerMRs.erase(Val: &SrcRT);
1301 }
1302 }
1303
1304 // If we're transfering to the default tracker we just need to delete the
1305 // tracked symbols for the source tracker.
1306 if (&DstRT == DefaultTracker.get()) {
1307 TrackerSymbols.erase(Val: &SrcRT);
1308 return;
1309 }
1310
1311 // If we're transferring from the default tracker we need to find all
1312 // currently untracked symbols.
1313 if (&SrcRT == DefaultTracker.get()) {
1314 assert(!TrackerSymbols.count(&SrcRT) &&
1315 "Default tracker should not appear in TrackerSymbols");
1316
1317 SymbolNameVector SymbolsToTrack;
1318
1319 SymbolNameSet CurrentlyTrackedSymbols;
1320 for (auto &KV : TrackerSymbols)
1321 CurrentlyTrackedSymbols.insert_range(R&: KV.second);
1322
1323 for (auto &KV : Symbols) {
1324 auto &Sym = KV.first;
1325 if (!CurrentlyTrackedSymbols.count(V: Sym))
1326 SymbolsToTrack.push_back(x: Sym);
1327 }
1328
1329 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1330 return;
1331 }
1332
1333 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1334
1335 // Finally if neither SrtRT or DstRT are the default tracker then
1336 // just append DstRT's tracked symbols to SrtRT's.
1337 auto SI = TrackerSymbols.find(Val: &SrcRT);
1338 if (SI == TrackerSymbols.end())
1339 return;
1340
1341 DstTrackedSymbols.reserve(n: DstTrackedSymbols.size() + SI->second.size());
1342 for (auto &Sym : SI->second)
1343 DstTrackedSymbols.push_back(x: std::move(Sym));
1344 TrackerSymbols.erase(I: SI);
1345}
1346
1347Error JITDylib::defineImpl(MaterializationUnit &MU) {
1348 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1349
1350 SymbolNameSet Duplicates;
1351 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1352 std::vector<SymbolStringPtr> MUDefsOverridden;
1353
1354 for (const auto &KV : MU.getSymbols()) {
1355 auto I = Symbols.find(Val: KV.first);
1356
1357 if (I != Symbols.end()) {
1358 if (KV.second.isStrong()) {
1359 if (I->second.getFlags().isStrong() ||
1360 I->second.getState() > SymbolState::NeverSearched)
1361 Duplicates.insert(V: KV.first);
1362 else {
1363 assert(I->second.getState() == SymbolState::NeverSearched &&
1364 "Overridden existing def should be in the never-searched "
1365 "state");
1366 ExistingDefsOverridden.push_back(x: KV.first);
1367 }
1368 } else
1369 MUDefsOverridden.push_back(x: KV.first);
1370 }
1371 }
1372
1373 // If there were any duplicate definitions then bail out.
1374 if (!Duplicates.empty()) {
1375 LLVM_DEBUG(
1376 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1377 return make_error<DuplicateDefinition>(Args: std::string(**Duplicates.begin()),
1378 Args: MU.getName().str());
1379 }
1380
1381 // Discard any overridden defs in this MU.
1382 LLVM_DEBUG({
1383 if (!MUDefsOverridden.empty())
1384 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1385 });
1386 for (auto &S : MUDefsOverridden)
1387 MU.doDiscard(JD: *this, Name: S);
1388
1389 // Discard existing overridden defs.
1390 LLVM_DEBUG({
1391 if (!ExistingDefsOverridden.empty())
1392 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1393 << "\n";
1394 });
1395 for (auto &S : ExistingDefsOverridden) {
1396
1397 auto UMII = UnmaterializedInfos.find(Val: S);
1398 assert(UMII != UnmaterializedInfos.end() &&
1399 "Overridden existing def should have an UnmaterializedInfo");
1400 UMII->second->MU->doDiscard(JD: *this, Name: S);
1401 }
1402
1403 // Finally, add the defs from this MU.
1404 for (auto &KV : MU.getSymbols()) {
1405 auto &SymEntry = Symbols[KV.first];
1406 SymEntry.setFlags(KV.second);
1407 SymEntry.setState(SymbolState::NeverSearched);
1408 SymEntry.setMaterializerAttached(true);
1409 }
1410
1411 return Error::success();
1412}
1413
1414void JITDylib::installMaterializationUnit(
1415 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1416
1417 /// defineImpl succeeded.
1418 if (&RT != DefaultTracker.get()) {
1419 auto &TS = TrackerSymbols[&RT];
1420 TS.reserve(n: TS.size() + MU->getSymbols().size());
1421 for (auto &KV : MU->getSymbols())
1422 TS.push_back(x: KV.first);
1423 }
1424
1425 auto UMI = std::make_shared<UnmaterializedInfo>(args: std::move(MU), args: &RT);
1426 for (auto &KV : UMI->MU->getSymbols())
1427 UnmaterializedInfos[KV.first] = UMI;
1428}
1429
1430void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1431 const SymbolNameSet &QuerySymbols) {
1432 for (auto &QuerySymbol : QuerySymbols) {
1433 auto MII = MaterializingInfos.find(Val: QuerySymbol);
1434 if (MII != MaterializingInfos.end())
1435 MII->second.removeQuery(Q);
1436 }
1437}
1438
1439Platform::~Platform() = default;
1440
1441Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
1442 ExecutionSession &ES,
1443 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1444
1445 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1446 Error CompoundErr = Error::success();
1447 std::mutex LookupMutex;
1448 std::condition_variable CV;
1449 uint64_t Count = InitSyms.size();
1450
1451 LLVM_DEBUG({
1452 dbgs() << "Issuing init-symbol lookup:\n";
1453 for (auto &KV : InitSyms)
1454 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1455 });
1456
1457 for (auto &KV : InitSyms) {
1458 auto *JD = KV.first;
1459 auto Names = std::move(KV.second);
1460 ES.lookup(
1461 K: LookupKind::Static,
1462 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1463 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1464 NotifyComplete: [&, JD](Expected<SymbolMap> Result) {
1465 std::lock_guard<std::mutex> Lock(LookupMutex);
1466 --Count;
1467 if (Result) {
1468 assert(!CompoundResult.count(JD) &&
1469 "Duplicate JITDylib in lookup?");
1470 CompoundResult[JD] = std::move(*Result);
1471 } else {
1472 CompoundErr =
1473 joinErrors(E1: std::move(CompoundErr), E2: Result.takeError());
1474 }
1475 CV.notify_one();
1476 },
1477 RegisterDependencies: NoDependenciesToRegister);
1478 }
1479
1480 std::unique_lock<std::mutex> Lock(LookupMutex);
1481 CV.wait(lock&: Lock, p: [&] { return Count == 0; });
1482
1483 if (CompoundErr)
1484 return std::move(CompoundErr);
1485
1486 return std::move(CompoundResult);
1487}
1488
1489void Platform::lookupInitSymbolsAsync(
1490 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1491 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1492
1493 class TriggerOnComplete {
1494 public:
1495 using OnCompleteFn = unique_function<void(Error)>;
1496 TriggerOnComplete(OnCompleteFn OnComplete)
1497 : OnComplete(std::move(OnComplete)) {}
1498 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1499 void reportResult(Error Err) {
1500 std::lock_guard<std::mutex> Lock(ResultMutex);
1501 LookupResult = joinErrors(E1: std::move(LookupResult), E2: std::move(Err));
1502 }
1503
1504 private:
1505 std::mutex ResultMutex;
1506 Error LookupResult{Error::success()};
1507 OnCompleteFn OnComplete;
1508 };
1509
1510 LLVM_DEBUG({
1511 dbgs() << "Issuing init-symbol lookup:\n";
1512 for (auto &KV : InitSyms)
1513 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1514 });
1515
1516 auto TOC = std::make_shared<TriggerOnComplete>(args: std::move(OnComplete));
1517
1518 for (auto &KV : InitSyms) {
1519 auto *JD = KV.first;
1520 auto Names = std::move(KV.second);
1521 ES.lookup(
1522 K: LookupKind::Static,
1523 SearchOrder: JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
1524 Symbols: std::move(Names), RequiredState: SymbolState::Ready,
1525 NotifyComplete: [TOC](Expected<SymbolMap> Result) {
1526 TOC->reportResult(Err: Result.takeError());
1527 },
1528 RegisterDependencies: NoDependenciesToRegister);
1529 }
1530}
1531
1532MaterializationTask::~MaterializationTask() {
1533 // If this task wasn't run then fail materialization.
1534 if (MR)
1535 MR->failMaterialization();
1536}
1537
1538void MaterializationTask::printDescription(raw_ostream &OS) {
1539 OS << "Materialization task: " << MU->getName() << " in "
1540 << MR->getTargetJITDylib().getName();
1541}
1542
1543void MaterializationTask::run() {
1544 assert(MU && "MU should not be null");
1545 assert(MR && "MR should not be null");
1546 MU->materialize(R: std::move(MR));
1547}
1548
1549void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1550
1551void LookupTask::run() { LS.continueLookup(Err: Error::success()); }
1552
1553ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1554 : EPC(std::move(EPC)), BootstrapJD(createBareJITDylib(Name: "<bootstrap>")) {
1555 // Associated EPC and this.
1556 this->EPC->ES = this;
1557 SymbolMap BootstrapSymbols;
1558 for (auto &[Name, Ptr] : this->EPC->getBootstrapSymbolsMap())
1559 BootstrapSymbols[intern(SymName: Name)] =
1560 ExecutorSymbolDef(Ptr, JITSymbolFlags::Exported);
1561 // Can't fail: BootstrapJD is a new, empty JD and the BootstrapSymbols
1562 // variable is a map, so can't contain duplicates.
1563 cantFail(Err: BootstrapJD.define(MU: absoluteSymbols(Symbols: std::move(BootstrapSymbols))));
1564
1565 // If the executor provides the ORC runtime's instance-symbol lookup then
1566 // make instance symbols available via the bootstrap JITDylib.
1567 //
1568 // TODO: Bind the lookup proxy with a weak lookupAndApply in BootstrapJD,
1569 // rather than searching the bootstrap symbols map. That isn't safe
1570 // yet: the blocking lookup materializes BootstrapJD's absolute
1571 // symbols via a task, so it would hang with a dispatcher that defers
1572 // tasks. To fix this we should add a version of absoluteSymbols that
1573 // is guaranteed to inject definitions directly into the symbol table
1574 // without spawning a task, and guarantee that blocking lookups return
1575 // without dispatching if all symbols are already fully materialized.
1576 Mangler Mangle(getTargetTriple());
1577 auto &BootstrapSymbolsMap = this->EPC->getBootstrapSymbolsMap();
1578 auto LookupFnI = BootstrapSymbolsMap.find(
1579 Key: Mangle.mangledCopy(Name: rt::sps_ci::SessionLookupInstanceSymbols::Name));
1580 if (LookupFnI != BootstrapSymbolsMap.end())
1581 BootstrapJD.addGenerator(
1582 DefGenerator: sps::createInstanceSymbolGenerator(LookupFnAddr: LookupFnI->second));
1583}
1584
1585ExecutionSession::~ExecutionSession() {
1586 // You must call endSession prior to destroying the session.
1587 assert(!SessionOpen &&
1588 "Session still open. Did you forget to call endSession?");
1589}
1590
1591Error ExecutionSession::endSession() {
1592 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1593
1594 WaitingOnGraph::OpRecorder *GOpRecorderToEnd = nullptr;
1595 auto JDsToRemove = runSessionLocked(F: [&] {
1596
1597#ifdef EXPENSIVE_CHECKS
1598 verifySessionState("Entering ExecutionSession::endSession");
1599#endif
1600
1601 if (SessionOpen)
1602 GOpRecorderToEnd = GOpRecorder;
1603 SessionOpen = false;
1604 return JDs;
1605 });
1606
1607 std::reverse(first: JDsToRemove.begin(), last: JDsToRemove.end());
1608
1609 auto Err = removeJITDylibs(JDsToRemove: std::move(JDsToRemove));
1610
1611 Err = joinErrors(E1: std::move(Err), E2: EPC->disconnect());
1612
1613 if (GOpRecorderToEnd)
1614 GOpRecorderToEnd->recordEnd();
1615
1616 return Err;
1617}
1618
1619void ExecutionSession::registerResourceManager(ResourceManager &RM) {
1620 runSessionLocked(F: [&] { ResourceManagers.push_back(x: &RM); });
1621}
1622
1623void ExecutionSession::deregisterResourceManager(ResourceManager &RM) {
1624 runSessionLocked(F: [&] {
1625 assert(!ResourceManagers.empty() && "No managers registered");
1626 if (ResourceManagers.back() == &RM)
1627 ResourceManagers.pop_back();
1628 else {
1629 auto I = llvm::find(Range&: ResourceManagers, Val: &RM);
1630 assert(I != ResourceManagers.end() && "RM not registered");
1631 ResourceManagers.erase(position: I);
1632 }
1633 });
1634}
1635
1636JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) {
1637 return runSessionLocked(F: [&, this]() -> JITDylib * {
1638 for (auto &JD : JDs)
1639 if (JD->getName() == Name)
1640 return JD.get();
1641 return nullptr;
1642 });
1643}
1644
1645JITDylib &ExecutionSession::createBareJITDylib(std::string Name) {
1646 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1647 return runSessionLocked(F: [&, this]() -> JITDylib & {
1648 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1649 JDs.push_back(x: new JITDylib(*this, std::move(Name)));
1650 return *JDs.back();
1651 });
1652}
1653
1654Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
1655 auto &JD = createBareJITDylib(Name);
1656 if (P)
1657 if (auto Err = P->setupJITDylib(JD))
1658 return std::move(Err);
1659 return JD;
1660}
1661
1662Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1663 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1664 runSessionLocked(F: [&] {
1665 for (auto &JD : JDsToRemove) {
1666 assert(JD->State == JITDylib::Open && "JD already closed");
1667 JD->State = JITDylib::Closing;
1668 auto I = llvm::find(Range&: JDs, Val: JD);
1669 assert(I != JDs.end() && "JD does not appear in session JDs");
1670 JDs.erase(position: I);
1671 }
1672 });
1673
1674 // Clear JITDylibs and notify the platform.
1675 Error Err = Error::success();
1676 for (auto JD : JDsToRemove) {
1677 Err = joinErrors(E1: std::move(Err), E2: JD->clear());
1678 if (P)
1679 Err = joinErrors(E1: std::move(Err), E2: P->teardownJITDylib(JD&: *JD));
1680 }
1681
1682 // Set JD to closed state. Clear remaining data structures.
1683 runSessionLocked(F: [&] {
1684 for (auto &JD : JDsToRemove) {
1685 assert(JD->State == JITDylib::Closing && "JD should be closing");
1686 JD->State = JITDylib::Closed;
1687 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1688 assert(JD->UnmaterializedInfos.empty() &&
1689 "JD.UnmaterializedInfos is not empty after clear");
1690 assert(JD->MaterializingInfos.empty() &&
1691 "JD.MaterializingInfos is not empty after clear");
1692 assert(JD->TrackerSymbols.empty() &&
1693 "TrackerSymbols is not empty after clear");
1694 JD->DefGenerators.clear();
1695 JD->LinkOrder.clear();
1696 }
1697 });
1698
1699 return Err;
1700}
1701
1702Expected<std::vector<JITDylibSP>>
1703JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1704 if (JDs.empty())
1705 return std::vector<JITDylibSP>();
1706
1707 auto &ES = JDs.front()->getExecutionSession();
1708 return ES.runSessionLocked(F: [&]() -> Expected<std::vector<JITDylibSP>> {
1709 DenseSet<JITDylib *> Visited;
1710 std::vector<JITDylibSP> Result;
1711
1712 for (auto &JD : JDs) {
1713
1714 if (JD->State != Open)
1715 return make_error<StringError>(
1716 Args: "Error building link order: " + JD->getName() + " is defunct",
1717 Args: inconvertibleErrorCode());
1718 if (Visited.count(V: JD.get()))
1719 continue;
1720
1721 SmallVector<JITDylibSP, 64> WorkStack;
1722 WorkStack.push_back(Elt: JD);
1723 Visited.insert(V: JD.get());
1724
1725 while (!WorkStack.empty()) {
1726 Result.push_back(x: std::move(WorkStack.back()));
1727 WorkStack.pop_back();
1728
1729 for (auto &KV : llvm::reverse(C&: Result.back()->LinkOrder)) {
1730 auto &JD = *KV.first;
1731 if (!Visited.insert(V: &JD).second)
1732 continue;
1733 WorkStack.push_back(Elt: &JD);
1734 }
1735 }
1736 }
1737 return Result;
1738 });
1739}
1740
1741Expected<std::vector<JITDylibSP>>
1742JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
1743 auto Result = getDFSLinkOrder(JDs);
1744 if (Result)
1745 std::reverse(first: Result->begin(), last: Result->end());
1746 return Result;
1747}
1748
1749Expected<std::vector<JITDylibSP>> JITDylib::getDFSLinkOrder() {
1750 return getDFSLinkOrder(JDs: {this});
1751}
1752
1753Expected<std::vector<JITDylibSP>> JITDylib::getReverseDFSLinkOrder() {
1754 return getReverseDFSLinkOrder(JDs: {this});
1755}
1756
1757void ExecutionSession::lookupFlags(
1758 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1759 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1760
1761 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1762 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1763 args: std::move(OnComplete)),
1764 Err: Error::success());
1765}
1766
1767Expected<SymbolFlagsMap>
1768ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
1769 SymbolLookupSet LookupSet) {
1770
1771 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1772 OL_applyQueryPhase1(IPLS: std::make_unique<InProgressLookupFlagsState>(
1773 args&: K, args: std::move(SearchOrder), args: std::move(LookupSet),
1774 args: [&ResultP](Expected<SymbolFlagsMap> Result) {
1775 ResultP.set_value(std::move(Result));
1776 }),
1777 Err: Error::success());
1778
1779 auto ResultF = ResultP.get_future();
1780 return ResultF.get();
1781}
1782
1783void ExecutionSession::lookup(
1784 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1785 SymbolLookupSet Symbols, SymbolState RequiredState,
1786 SymbolsResolvedCallback NotifyComplete,
1787 RegisterDependenciesFunction RegisterDependencies) {
1788
1789 LLVM_DEBUG({
1790 runSessionLocked([&]() {
1791 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1792 << " (required state: " << RequiredState << ")\n";
1793 });
1794 });
1795
1796 // lookup can be re-entered recursively if running on a single thread. Run any
1797 // outstanding MUs in case this query depends on them, otherwise this lookup
1798 // will starve waiting for a result from an MU that is stuck in the queue.
1799 dispatchOutstandingMUs();
1800
1801 auto Unresolved = std::move(Symbols);
1802 auto Q = std::make_shared<AsynchronousSymbolQuery>(args&: Unresolved, args&: RequiredState,
1803 args: std::move(NotifyComplete));
1804
1805 auto IPLS = std::make_unique<InProgressFullLookupState>(
1806 args&: K, args: SearchOrder, args: std::move(Unresolved), args&: RequiredState, args: std::move(Q),
1807 args: std::move(RegisterDependencies));
1808
1809 OL_applyQueryPhase1(IPLS: std::move(IPLS), Err: Error::success());
1810}
1811
1812Expected<SymbolMap>
1813ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1814 SymbolLookupSet Symbols, LookupKind K,
1815 SymbolState RequiredState,
1816 RegisterDependenciesFunction RegisterDependencies) {
1817#if LLVM_ENABLE_THREADS
1818 // In the threaded case we use promises to return the results.
1819 std::promise<MSVCPExpected<SymbolMap>> PromisedResult;
1820
1821 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1822 PromisedResult.set_value(std::move(R));
1823 };
1824
1825#else
1826 SymbolMap Result;
1827 Error ResolutionError = Error::success();
1828
1829 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1830 ErrorAsOutParameter _(ResolutionError);
1831 if (R)
1832 Result = std::move(*R);
1833 else
1834 ResolutionError = R.takeError();
1835 };
1836#endif
1837
1838 // Perform the asynchronous lookup.
1839 lookup(K, SearchOrder, Symbols: std::move(Symbols), RequiredState,
1840 NotifyComplete: std::move(NotifyComplete), RegisterDependencies);
1841
1842#if LLVM_ENABLE_THREADS
1843 return PromisedResult.get_future().get();
1844#else
1845 if (ResolutionError)
1846 return std::move(ResolutionError);
1847
1848 return Result;
1849#endif
1850}
1851
1852Expected<ExecutorSymbolDef>
1853ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder,
1854 SymbolStringPtr Name, SymbolState RequiredState) {
1855 SymbolLookupSet Names({Name});
1856
1857 if (auto ResultMap = lookup(SearchOrder, Symbols: std::move(Names), K: LookupKind::Static,
1858 RequiredState, RegisterDependencies: NoDependenciesToRegister)) {
1859 assert(ResultMap->size() == 1 && "Unexpected number of results");
1860 assert(ResultMap->count(Name) && "Missing result for symbol");
1861 return std::move(ResultMap->begin()->second);
1862 } else
1863 return ResultMap.takeError();
1864}
1865
1866Expected<ExecutorSymbolDef>
1867ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name,
1868 SymbolState RequiredState) {
1869 return lookup(SearchOrder: makeJITDylibSearchOrder(JDs: SearchOrder), Name, RequiredState);
1870}
1871
1872Expected<ExecutorSymbolDef>
1873ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name,
1874 SymbolState RequiredState) {
1875 return lookup(SearchOrder, Name: intern(SymName: Name), RequiredState);
1876}
1877
1878Error ExecutionSession::registerCallControllerHandlers(
1879 JITDylib &JD, std::vector<CallControllerHandlerBinding> Hs) {
1880
1881 // Mangle and intern the tag names, and take the handlers.
1882 Mangler Mangle(getTargetTriple());
1883 DenseMap<SymbolStringPtr, CallControllerHandlerFn> Handlers;
1884 SymbolLookupSet LookupSet;
1885 for (auto &H : Hs) {
1886 auto TagName = Mangle.withMangledNameDo(
1887 H: [&](StringRef N) { return intern(SymName: N); }, NameSpec: H.getName());
1888 auto Handler = H.takeHandler();
1889 assert(Handler && "CallControllerHandler implementation missing");
1890 if (!Handlers.try_emplace(Key: TagName, Args: std::move(Handler)).second)
1891 return make_error<StringError>(
1892 Args: "Duplicate call-controller handler binding for tag " + *TagName,
1893 Args: inconvertibleErrorCode());
1894 LookupSet.add(Name: std::move(TagName), Flags: H.getLookupFlags());
1895 }
1896
1897 auto TagSyms = lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1898 Symbols: std::move(LookupSet));
1899 if (!TagSyms)
1900 return TagSyms.takeError();
1901
1902 // Associate tag addresses with implementations.
1903 std::lock_guard<std::mutex> Lock(CallControllerHandlersMutex);
1904
1905 // Check that no tags are being overwritten.
1906 for (auto &[TagName, TagSym] : *TagSyms) {
1907 auto TagAddr = TagSym.getAddress();
1908 if (CallControllerHandlers.count(Val: TagAddr))
1909 return make_error<StringError>(Args: "Tag " + formatv(Fmt: "{0:x}", Vals&: TagAddr) +
1910 " (for " + *TagName +
1911 ") already registered",
1912 Args: inconvertibleErrorCode());
1913 }
1914
1915 // At this point we're guaranteed to succeed. Install the handlers. Handlers
1916 // for weakly referenced tags that weren't found are dropped.
1917 for (auto &[TagName, TagSym] : *TagSyms) {
1918 auto TagAddr = TagSym.getAddress();
1919 auto I = Handlers.find(Val: TagName);
1920 assert(I != Handlers.end() && "No handler for tag in lookup result");
1921 CallControllerHandlers[TagAddr] =
1922 std::make_unique<CallControllerHandlerFn>(args: std::move(I->second));
1923 LLVM_DEBUG({
1924 dbgs() << "Registered call-controller handler for tag \"" << *TagName
1925 << "\" (" << formatv("{0:x}", TagAddr) << ")\n";
1926 });
1927 }
1928
1929 return Error::success();
1930}
1931
1932void ExecutionSession::runCallControllerHandler(
1933 CallControllerReturnFn Return, ExecutorAddr HandlerFnTagAddr,
1934 shared::WrapperFunctionBuffer ArgBytes) {
1935
1936 // Handlers are never removed, so H remains valid after the lock is
1937 // released.
1938 CallControllerHandlerFn *H = nullptr;
1939 {
1940 std::lock_guard<std::mutex> Lock(CallControllerHandlersMutex);
1941 auto I = CallControllerHandlers.find(Val: HandlerFnTagAddr);
1942 if (I != CallControllerHandlers.end())
1943 H = I->second.get();
1944 }
1945
1946 if (!H)
1947 return Return(shared::WrapperFunctionBuffer::createOutOfBandError(
1948 Msg: ("No call-controller handler registered for tag " +
1949 formatv(Fmt: "{0:x16}", Vals&: HandlerFnTagAddr))
1950 .str()));
1951
1952 (*H)(std::move(Return), std::move(ArgBytes));
1953}
1954
1955void ExecutionSession::dump(raw_ostream &OS) {
1956 runSessionLocked(F: [this, &OS]() {
1957 for (auto &JD : JDs)
1958 JD->dump(OS);
1959 });
1960}
1961
1962#ifdef EXPENSIVE_CHECKS
1963bool ExecutionSession::verifySessionState(Twine Phase) {
1964 return runSessionLocked([&]() {
1965 bool AllOk = true;
1966
1967 for (auto &JD : JDs) {
1968
1969 auto LogFailure = [&]() -> raw_fd_ostream & {
1970 auto &Stream = errs();
1971 if (AllOk)
1972 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1973 << "\n";
1974 Stream << " In JITDylib " << JD->getName() << ", ";
1975 AllOk = false;
1976 return Stream;
1977 };
1978
1979 if (JD->State != JITDylib::Open) {
1980 LogFailure()
1981 << "state is not Open, but JD is in ExecutionSession list.";
1982 }
1983
1984 // Check symbol table.
1985 // 1. If the entry state isn't resolved then check that no address has
1986 // been set.
1987 // 2. Check that if the hasMaterializerAttached flag is set then there is
1988 // an UnmaterializedInfo entry, and vice-versa.
1989 for (auto &[Sym, Entry] : JD->Symbols) {
1990 // Check that unresolved symbols have null addresses.
1991 if (Entry.getState() < SymbolState::Resolved) {
1992 if (Entry.getAddress()) {
1993 LogFailure() << "symbol " << Sym << " has state "
1994 << Entry.getState()
1995 << " (not-yet-resolved) but non-null address "
1996 << Entry.getAddress() << ".\n";
1997 }
1998 }
1999
2000 // Check that the hasMaterializerAttached flag is correct.
2001 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2002 if (Entry.hasMaterializerAttached()) {
2003 if (UMIItr == JD->UnmaterializedInfos.end()) {
2004 LogFailure() << "symbol " << Sym
2005 << " entry claims materializer attached, but "
2006 "UnmaterializedInfos has no corresponding entry.\n";
2007 }
2008 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2009 LogFailure()
2010 << "symbol " << Sym
2011 << " entry claims no materializer attached, but "
2012 "UnmaterializedInfos has an unexpected entry for it.\n";
2013 }
2014 }
2015
2016 // Check that every UnmaterializedInfo entry has a corresponding entry
2017 // in the Symbols table.
2018 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2019 auto SymItr = JD->Symbols.find(Sym);
2020 if (SymItr == JD->Symbols.end()) {
2021 LogFailure()
2022 << "symbol " << Sym
2023 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2024 }
2025 }
2026
2027 // Check consistency of the MaterializingInfos table.
2028 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2029
2030 auto SymItr = JD->Symbols.find(Sym);
2031 if (SymItr == JD->Symbols.end()) {
2032 // If there's no Symbols entry for this MaterializingInfos entry then
2033 // report that.
2034 LogFailure()
2035 << "symbol " << Sym
2036 << " has MaterializingInfos entry, but no Symbols entry.\n";
2037 } else {
2038 // Otherwise check consistency between Symbols and MaterializingInfos.
2039
2040 // Ready symbols should not have MaterializingInfos.
2041 if (SymItr->second.getState() == SymbolState::Ready) {
2042 LogFailure()
2043 << "symbol " << Sym
2044 << " is in Ready state, should not have MaterializingInfo.\n";
2045 }
2046
2047 // Pending queries should be for subsequent states.
2048 auto CurState = static_cast<SymbolState>(
2049 static_cast<std::underlying_type_t<SymbolState>>(
2050 SymItr->second.getState()) + 1);
2051 for (auto &Q : MII.PendingQueries) {
2052 if (Q->getRequiredState() != CurState) {
2053 if (Q->getRequiredState() > CurState)
2054 CurState = Q->getRequiredState();
2055 else
2056 LogFailure() << "symbol " << Sym
2057 << " has stale or misordered queries.\n";
2058 }
2059 }
2060 }
2061 }
2062 }
2063
2064 return AllOk;
2065 });
2066}
2067#endif // EXPENSIVE_CHECKS
2068
2069void ExecutionSession::dispatchOutstandingMUs() {
2070 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2071 while (true) {
2072 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2073 std::unique_ptr<MaterializationResponsibility>>>
2074 JMU;
2075
2076 {
2077 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2078 if (!OutstandingMUs.empty()) {
2079 JMU.emplace(args: std::move(OutstandingMUs.back()));
2080 OutstandingMUs.pop_back();
2081 }
2082 }
2083
2084 if (!JMU)
2085 break;
2086
2087 assert(JMU->first && "No MU?");
2088 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2089 dispatchTask(T: std::make_unique<MaterializationTask>(args: std::move(JMU->first),
2090 args: std::move(JMU->second)));
2091 }
2092 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2093}
2094
2095Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2096 LLVM_DEBUG({
2097 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2098 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2099 });
2100 std::vector<ResourceManager *> CurrentResourceManagers;
2101
2102 JITDylib::RemoveTrackerResult R;
2103
2104 runSessionLocked(F: [&] {
2105 CurrentResourceManagers = ResourceManagers;
2106 RT.makeDefunct();
2107 R = RT.getJITDylib().IL_removeTracker(RT);
2108 });
2109
2110 // Release any defunct MaterializationUnits.
2111 R.DefunctMUs.clear();
2112
2113 Error Err = Error::success();
2114
2115 auto &JD = RT.getJITDylib();
2116 for (auto *L : reverse(C&: CurrentResourceManagers))
2117 Err = joinErrors(E1: std::move(Err),
2118 E2: L->handleRemoveResources(JD, K: RT.getKeyUnsafe()));
2119
2120 for (auto &Q : R.QueriesToFail)
2121 Q->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2122 Args&: R.FailedSymbols));
2123
2124 return Err;
2125}
2126
2127void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2128 ResourceTracker &SrcRT) {
2129 LLVM_DEBUG({
2130 dbgs() << "In " << SrcRT.getJITDylib().getName()
2131 << " transfering resources from tracker "
2132 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2133 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2134 });
2135
2136 // No-op transfers are allowed and do not invalidate the source.
2137 if (&DstRT == &SrcRT)
2138 return;
2139
2140 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2141 "Can't transfer resources between JITDylibs");
2142 runSessionLocked(F: [&]() {
2143 SrcRT.makeDefunct();
2144 auto &JD = DstRT.getJITDylib();
2145 JD.transferTracker(DstRT, SrcRT);
2146 for (auto *L : reverse(C&: ResourceManagers))
2147 L->handleTransferResources(JD, DstK: DstRT.getKeyUnsafe(),
2148 SrcK: SrcRT.getKeyUnsafe());
2149 });
2150}
2151
2152void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2153 runSessionLocked(F: [&]() {
2154 LLVM_DEBUG({
2155 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2156 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2157 });
2158 if (!RT.isDefunct())
2159 transferResourceTracker(DstRT&: *RT.getJITDylib().getDefaultResourceTracker(),
2160 SrcRT&: RT);
2161 });
2162}
2163
2164Error ExecutionSession::IL_updateCandidatesFor(
2165 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2166 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2167 return Candidates.forEachWithRemoval(
2168 Body: [&](const SymbolStringPtr &Name,
2169 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2170 /// Search for the symbol. If not found then continue without
2171 /// removal.
2172 auto SymI = JD.Symbols.find(Val: Name);
2173 if (SymI == JD.Symbols.end())
2174 return false;
2175
2176 // If this is a non-exported symbol and we're matching exported
2177 // symbols only then remove this symbol from the candidates list.
2178 //
2179 // If we're tracking non-candidates then add this to the non-candidate
2180 // list.
2181 if (!SymI->second.getFlags().isExported() &&
2182 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2183 if (NonCandidates)
2184 NonCandidates->add(Name, Flags: SymLookupFlags);
2185 return true;
2186 }
2187
2188 // If we match against a materialization-side-effects only symbol
2189 // then make sure it is weakly-referenced. Otherwise bail out with
2190 // an error.
2191 // FIXME: Use a "materialization-side-effects-only symbols must be
2192 // weakly referenced" specific error here to reduce confusion.
2193 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2194 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
2195 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2196 Args: SymbolNameVector({Name}));
2197
2198 // If we matched against this symbol but it is in the error state
2199 // then bail out and treat it as a failure to materialize.
2200 if (SymI->second.getFlags().hasError()) {
2201 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2202 (*FailedSymbolsMap)[&JD] = {Name};
2203 return make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
2204 Args: std::move(FailedSymbolsMap));
2205 }
2206
2207 // Otherwise this is a match. Remove it from the candidate set.
2208 return true;
2209 });
2210}
2211
2212void ExecutionSession::OL_resumeLookupAfterGeneration(
2213 InProgressLookupState &IPLS) {
2214
2215 assert(IPLS.GenState != InProgressLookupState::NotInGenerator &&
2216 "Should not be called for not-in-generator lookups");
2217 IPLS.GenState = InProgressLookupState::NotInGenerator;
2218
2219 LookupState LS;
2220
2221 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2222 IPLS.CurDefGeneratorStack.pop_back();
2223 std::lock_guard<std::mutex> Lock(DG->M);
2224
2225 // If there are no pending lookups then mark the generator as free and
2226 // return.
2227 if (DG->PendingLookups.empty()) {
2228 DG->InUse = false;
2229 return;
2230 }
2231
2232 // Otherwise resume the next lookup.
2233 LS = std::move(DG->PendingLookups.front());
2234 DG->PendingLookups.pop_front();
2235 }
2236
2237 if (LS.IPLS) {
2238 LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator;
2239 dispatchTask(T: std::make_unique<LookupTask>(args: std::move(LS)));
2240 }
2241}
2242
2243void ExecutionSession::OL_applyQueryPhase1(
2244 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2245
2246 LLVM_DEBUG({
2247 dbgs() << "Entering OL_applyQueryPhase1:\n"
2248 << " Lookup kind: " << IPLS->K << "\n"
2249 << " Search order: " << IPLS->SearchOrder
2250 << ", Current index = " << IPLS->CurSearchOrderIndex
2251 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2252 << " Lookup set: " << IPLS->LookupSet << "\n"
2253 << " Definition generator candidates: "
2254 << IPLS->DefGeneratorCandidates << "\n"
2255 << " Definition generator non-candidates: "
2256 << IPLS->DefGeneratorNonCandidates << "\n";
2257 });
2258
2259 if (IPLS->GenState == InProgressLookupState::InGenerator)
2260 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2261
2262 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2263 "Lookup should not be in InGenerator state here");
2264
2265 // FIXME: We should attach the query as we go: This provides a result in a
2266 // single pass in the common case where all symbols have already reached the
2267 // required state. The query could be detached again in the 'fail' method on
2268 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2269
2270 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2271
2272 // If we've been handed an error or received one back from a generator then
2273 // fail the query. We don't need to unlink: At this stage the query hasn't
2274 // actually been lodged.
2275 if (Err)
2276 return IPLS->fail(Err: std::move(Err));
2277
2278 // Get the next JITDylib and lookup flags.
2279 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2280 auto &JD = *KV.first;
2281 auto JDLookupFlags = KV.second;
2282
2283 LLVM_DEBUG({
2284 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2285 << ") with lookup set " << IPLS->LookupSet << ":\n";
2286 });
2287
2288 // If we've just reached a new JITDylib then perform some setup.
2289 if (IPLS->NewJITDylib) {
2290 // Add any non-candidates from the last JITDylib (if any) back on to the
2291 // list of definition candidates for this JITDylib, reset definition
2292 // non-candidates to the empty set.
2293 SymbolLookupSet Tmp;
2294 std::swap(a&: IPLS->DefGeneratorNonCandidates, b&: Tmp);
2295 IPLS->DefGeneratorCandidates.append(Other: std::move(Tmp));
2296
2297 LLVM_DEBUG({
2298 dbgs() << " First time visiting " << JD.getName()
2299 << ", resetting candidate sets and building generator stack\n";
2300 });
2301
2302 // Build the definition generator stack for this JITDylib.
2303 runSessionLocked(F: [&] {
2304 IPLS->CurDefGeneratorStack.reserve(n: JD.DefGenerators.size());
2305 llvm::append_range(C&: IPLS->CurDefGeneratorStack,
2306 R: reverse(C&: JD.DefGenerators));
2307 });
2308
2309 // Flag that we've done our initialization.
2310 IPLS->NewJITDylib = false;
2311 }
2312
2313 // Remove any generation candidates that are already defined (and match) in
2314 // this JITDylib.
2315 runSessionLocked(F: [&] {
2316 // Update the list of candidates (and non-candidates) for definition
2317 // generation.
2318 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2319 Err = IL_updateCandidatesFor(
2320 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2321 NonCandidates: JD.DefGenerators.empty() ? nullptr
2322 : &IPLS->DefGeneratorNonCandidates);
2323 LLVM_DEBUG({
2324 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2325 << "\n";
2326 });
2327
2328 // If this lookup was resumed after auto-suspension but all candidates
2329 // have already been generated (by some previous call to the generator)
2330 // treat the lookup as if it had completed generation.
2331 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2332 IPLS->DefGeneratorCandidates.empty())
2333 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2334 });
2335
2336 // If we encountered an error while filtering generation candidates then
2337 // bail out.
2338 if (Err)
2339 return IPLS->fail(Err: std::move(Err));
2340
2341 /// Apply any definition generators on the stack.
2342 LLVM_DEBUG({
2343 if (IPLS->CurDefGeneratorStack.empty())
2344 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2345 else if (IPLS->DefGeneratorCandidates.empty())
2346 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2347 else
2348 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2349 << " remaining generators for "
2350 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2351 });
2352 while (!IPLS->CurDefGeneratorStack.empty() &&
2353 !IPLS->DefGeneratorCandidates.empty()) {
2354 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2355
2356 if (!DG)
2357 return IPLS->fail(Err: make_error<StringError>(
2358 Args: "DefinitionGenerator removed while lookup in progress",
2359 Args: inconvertibleErrorCode()));
2360
2361 // At this point the lookup is in either the NotInGenerator state, or in
2362 // the ResumedForGenerator state.
2363 // If this lookup is in the NotInGenerator state then check whether the
2364 // generator is in use. If the generator is not in use then move the
2365 // lookup to the InGenerator state and continue. If the generator is
2366 // already in use then just add this lookup to the pending lookups list
2367 // and bail out.
2368 // If this lookup is in the ResumedForGenerator state then just move it
2369 // to InGenerator and continue.
2370 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2371 std::lock_guard<std::mutex> Lock(DG->M);
2372 if (DG->InUse) {
2373 DG->PendingLookups.push_back(x: std::move(IPLS));
2374 return;
2375 }
2376 DG->InUse = true;
2377 }
2378
2379 IPLS->GenState = InProgressLookupState::InGenerator;
2380
2381 auto K = IPLS->K;
2382 auto &LookupSet = IPLS->DefGeneratorCandidates;
2383
2384 // Run the generator. If the generator takes ownership of QA then this
2385 // will break the loop.
2386 {
2387 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2388 LookupState LS(std::move(IPLS));
2389 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2390 IPLS = std::move(LS.IPLS);
2391 }
2392
2393 // If the lookup returned then pop the generator stack and unblock the
2394 // next lookup on this generator (if any).
2395 if (IPLS)
2396 OL_resumeLookupAfterGeneration(IPLS&: *IPLS);
2397
2398 // If there was an error then fail the query.
2399 if (Err) {
2400 LLVM_DEBUG({
2401 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2402 });
2403 assert(IPLS && "LS cannot be retained if error is returned");
2404 return IPLS->fail(Err: std::move(Err));
2405 }
2406
2407 // Otherwise if QA was captured then break the loop.
2408 if (!IPLS) {
2409 LLVM_DEBUG(
2410 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2411 return;
2412 }
2413
2414 // Otherwise if we're continuing around the loop then update candidates
2415 // for the next round.
2416 runSessionLocked(F: [&] {
2417 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2418 Err = IL_updateCandidatesFor(
2419 JD, JDLookupFlags, Candidates&: IPLS->DefGeneratorCandidates,
2420 NonCandidates: JD.DefGenerators.empty() ? nullptr
2421 : &IPLS->DefGeneratorNonCandidates);
2422 });
2423
2424 // If updating candidates failed then fail the query.
2425 if (Err) {
2426 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2427 return IPLS->fail(Err: std::move(Err));
2428 }
2429 }
2430
2431 if (IPLS->DefGeneratorCandidates.empty() &&
2432 IPLS->DefGeneratorNonCandidates.empty()) {
2433 // Early out if there are no remaining symbols.
2434 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2435 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2436 break;
2437 } else {
2438 // If we get here then we've moved on to the next JITDylib with candidates
2439 // remaining.
2440 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2441 ++IPLS->CurSearchOrderIndex;
2442 IPLS->NewJITDylib = true;
2443 }
2444 }
2445
2446 // Remove any weakly referenced candidates that could not be found/generated.
2447 IPLS->DefGeneratorCandidates.remove_if(
2448 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2449 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2450 });
2451
2452 // If we get here then we've finished searching all JITDylibs.
2453 // If we matched all symbols then move to phase 2, otherwise fail the query
2454 // with a SymbolsNotFound error.
2455 if (IPLS->DefGeneratorCandidates.empty()) {
2456 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2457 IPLS->complete(IPLS: std::move(IPLS));
2458 } else {
2459 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2460 IPLS->fail(Err: make_error<SymbolsNotFound>(
2461 Args: getSymbolStringPool(), Args: IPLS->DefGeneratorCandidates.getSymbolNames()));
2462 }
2463}
2464
2465void ExecutionSession::OL_completeLookup(
2466 std::unique_ptr<InProgressLookupState> IPLS,
2467 std::shared_ptr<AsynchronousSymbolQuery> Q,
2468 RegisterDependenciesFunction RegisterDependencies) {
2469
2470 LLVM_DEBUG({
2471 dbgs() << "Entering OL_completeLookup:\n"
2472 << " Lookup kind: " << IPLS->K << "\n"
2473 << " Search order: " << IPLS->SearchOrder
2474 << ", Current index = " << IPLS->CurSearchOrderIndex
2475 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2476 << " Lookup set: " << IPLS->LookupSet << "\n"
2477 << " Definition generator candidates: "
2478 << IPLS->DefGeneratorCandidates << "\n"
2479 << " Definition generator non-candidates: "
2480 << IPLS->DefGeneratorNonCandidates << "\n";
2481 });
2482
2483 bool QueryComplete = false;
2484 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2485
2486 auto LodgingErr = runSessionLocked(F: [&]() -> Error {
2487 for (auto &KV : IPLS->SearchOrder) {
2488 auto &JD = *KV.first;
2489 auto JDLookupFlags = KV.second;
2490 LLVM_DEBUG({
2491 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2492 << ") with lookup set " << IPLS->LookupSet << ":\n";
2493 });
2494
2495 auto Err = IPLS->LookupSet.forEachWithRemoval(
2496 Body: [&](const SymbolStringPtr &Name,
2497 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2498 LLVM_DEBUG({
2499 dbgs() << " Attempting to match \"" << Name << "\" ("
2500 << SymLookupFlags << ")... ";
2501 });
2502
2503 /// Search for the symbol. If not found then continue without
2504 /// removal.
2505 auto SymI = JD.Symbols.find(Val: Name);
2506 if (SymI == JD.Symbols.end()) {
2507 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2508 return false;
2509 }
2510
2511 // If this is a non-exported symbol and we're matching exported
2512 // symbols only then skip this symbol without removal.
2513 if (!SymI->second.getFlags().isExported() &&
2514 JDLookupFlags ==
2515 JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2516 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2517 return false;
2518 }
2519
2520 // If we match against a materialization-side-effects only symbol
2521 // then make sure it is weakly-referenced. Otherwise bail out with
2522 // an error.
2523 // FIXME: Use a "materialization-side-effects-only symbols must be
2524 // weakly referenced" specific error here to reduce confusion.
2525 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2526 SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) {
2527 LLVM_DEBUG({
2528 dbgs() << "error: "
2529 "required, but symbol is has-side-effects-only\n";
2530 });
2531 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2532 Args: SymbolNameVector({Name}));
2533 }
2534
2535 // If we matched against this symbol but it is in the error state
2536 // then bail out and treat it as a failure to materialize.
2537 if (SymI->second.getFlags().hasError()) {
2538 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2539 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2540 (*FailedSymbolsMap)[&JD] = {Name};
2541 return make_error<FailedToMaterialize>(
2542 Args: getSymbolStringPool(), Args: std::move(FailedSymbolsMap));
2543 }
2544
2545 // Otherwise this is a match.
2546
2547 // If this symbol is already in the required state then notify the
2548 // query, remove the symbol and continue.
2549 if (SymI->second.getState() >= Q->getRequiredState()) {
2550 LLVM_DEBUG(dbgs()
2551 << "matched, symbol already in required state\n");
2552 Q->notifySymbolMetRequiredState(Name, Sym: SymI->second.getSymbol());
2553
2554 // If this symbol is in anything other than the Ready state then
2555 // we need to track the dependence.
2556 if (SymI->second.getState() != SymbolState::Ready)
2557 Q->addQueryDependence(JD, Name);
2558
2559 return true;
2560 }
2561
2562 // Otherwise this symbol does not yet meet the required state. Check
2563 // whether it has a materializer attached, and if so prepare to run
2564 // it.
2565 if (SymI->second.hasMaterializerAttached()) {
2566 assert(SymI->second.getAddress() == ExecutorAddr() &&
2567 "Symbol not resolved but already has address?");
2568 auto UMII = JD.UnmaterializedInfos.find(Val: Name);
2569 assert(UMII != JD.UnmaterializedInfos.end() &&
2570 "Lazy symbol should have UnmaterializedInfo");
2571
2572 auto UMI = UMII->second;
2573 assert(UMI->MU && "Materializer should not be null");
2574 assert(UMI->RT && "Tracker should not be null");
2575 LLVM_DEBUG({
2576 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2577 << " (" << UMI->MU->getName() << ")\n";
2578 });
2579
2580 // Move all symbols associated with this MaterializationUnit into
2581 // materializing state.
2582 for (auto &KV : UMI->MU->getSymbols()) {
2583 auto SymK = JD.Symbols.find(Val: KV.first);
2584 assert(SymK != JD.Symbols.end() &&
2585 "No entry for symbol covered by MaterializationUnit");
2586 SymK->second.setMaterializerAttached(false);
2587 SymK->second.setState(SymbolState::Materializing);
2588 JD.UnmaterializedInfos.erase(Val: KV.first);
2589 }
2590
2591 // Add MU to the list of MaterializationUnits to be materialized.
2592 CollectedUMIs[&JD].push_back(x: std::move(UMI));
2593 } else
2594 LLVM_DEBUG(dbgs() << "matched, registering query");
2595
2596 // Add the query to the PendingQueries list and continue, deleting
2597 // the element from the lookup set.
2598 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2599 SymI->second.getState() != SymbolState::Ready &&
2600 "By this line the symbol should be materializing");
2601 auto &MI = JD.MaterializingInfos[Name];
2602 MI.addQuery(Q);
2603 Q->addQueryDependence(JD, Name);
2604
2605 return true;
2606 });
2607
2608 JD.shrinkMaterializationInfoMemory();
2609
2610 // Handle failure.
2611 if (Err) {
2612
2613 LLVM_DEBUG({
2614 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2615 });
2616
2617 // Detach the query.
2618 Q->detach();
2619
2620 // Replace the MUs.
2621 for (auto &KV : CollectedUMIs) {
2622 auto &JD = *KV.first;
2623 for (auto &UMI : KV.second)
2624 for (auto &KV2 : UMI->MU->getSymbols()) {
2625 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2626 "Unexpected materializer in map");
2627 auto SymI = JD.Symbols.find(Val: KV2.first);
2628 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2629 assert(SymI->second.getState() == SymbolState::Materializing &&
2630 "Can not replace symbol that is not materializing");
2631 assert(!SymI->second.hasMaterializerAttached() &&
2632 "MaterializerAttached flag should not be set");
2633 SymI->second.setMaterializerAttached(true);
2634 JD.UnmaterializedInfos[KV2.first] = UMI;
2635 }
2636 }
2637
2638 return Err;
2639 }
2640 }
2641
2642 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2643 IPLS->LookupSet.forEachWithRemoval(
2644 Body: [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2645 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2646 Q->dropSymbol(Name);
2647 return true;
2648 } else
2649 return false;
2650 });
2651
2652 if (!IPLS->LookupSet.empty()) {
2653 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2654 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2655 Args: IPLS->LookupSet.getSymbolNames());
2656 }
2657
2658 // Record whether the query completed.
2659 QueryComplete = Q->isComplete();
2660
2661 LLVM_DEBUG({
2662 dbgs() << "Query successfully "
2663 << (QueryComplete ? "completed" : "lodged") << "\n";
2664 });
2665
2666 // Move the collected MUs to the OutstandingMUs list.
2667 if (!CollectedUMIs.empty()) {
2668 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2669
2670 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2671 for (auto &KV : CollectedUMIs) {
2672 LLVM_DEBUG({
2673 auto &JD = *KV.first;
2674 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2675 << " MUs.\n";
2676 });
2677 for (auto &UMI : KV.second) {
2678 auto MR = createMaterializationResponsibility(
2679 RT&: *UMI->RT, Symbols: std::move(UMI->MU->SymbolFlags),
2680 InitSymbol: std::move(UMI->MU->InitSymbol));
2681 OutstandingMUs.push_back(
2682 x: std::make_pair(x: std::move(UMI->MU), y: std::move(MR)));
2683 }
2684 }
2685 } else
2686 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2687
2688 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2689 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2690 RegisterDependencies(Q->QueryRegistrations);
2691 } else
2692 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2693
2694 return Error::success();
2695 });
2696
2697 if (LodgingErr) {
2698 LLVM_DEBUG(dbgs() << "Failing query\n");
2699 Q->detach();
2700 Q->handleFailed(Err: std::move(LodgingErr));
2701 return;
2702 }
2703
2704 if (QueryComplete) {
2705 LLVM_DEBUG(dbgs() << "Completing query\n");
2706 Q->handleComplete(ES&: *this);
2707 }
2708
2709 dispatchOutstandingMUs();
2710}
2711
2712void ExecutionSession::OL_completeLookupFlags(
2713 std::unique_ptr<InProgressLookupState> IPLS,
2714 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2715
2716 auto Result = runSessionLocked(F: [&]() -> Expected<SymbolFlagsMap> {
2717 LLVM_DEBUG({
2718 dbgs() << "Entering OL_completeLookupFlags:\n"
2719 << " Lookup kind: " << IPLS->K << "\n"
2720 << " Search order: " << IPLS->SearchOrder
2721 << ", Current index = " << IPLS->CurSearchOrderIndex
2722 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2723 << " Lookup set: " << IPLS->LookupSet << "\n"
2724 << " Definition generator candidates: "
2725 << IPLS->DefGeneratorCandidates << "\n"
2726 << " Definition generator non-candidates: "
2727 << IPLS->DefGeneratorNonCandidates << "\n";
2728 });
2729
2730 SymbolFlagsMap Result;
2731
2732 // Attempt to find flags for each symbol.
2733 for (auto &KV : IPLS->SearchOrder) {
2734 auto &JD = *KV.first;
2735 auto JDLookupFlags = KV.second;
2736 LLVM_DEBUG({
2737 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2738 << ") with lookup set " << IPLS->LookupSet << ":\n";
2739 });
2740
2741 IPLS->LookupSet.forEachWithRemoval(Body: [&](const SymbolStringPtr &Name,
2742 SymbolLookupFlags SymLookupFlags) {
2743 LLVM_DEBUG({
2744 dbgs() << " Attempting to match \"" << Name << "\" ("
2745 << SymLookupFlags << ")... ";
2746 });
2747
2748 // Search for the symbol. If not found then continue without removing
2749 // from the lookup set.
2750 auto SymI = JD.Symbols.find(Val: Name);
2751 if (SymI == JD.Symbols.end()) {
2752 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2753 return false;
2754 }
2755
2756 // If this is a non-exported symbol then it doesn't match. Skip it.
2757 if (!SymI->second.getFlags().isExported() &&
2758 JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) {
2759 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2760 return false;
2761 }
2762
2763 LLVM_DEBUG({
2764 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2765 << "\n";
2766 });
2767 Result[Name] = SymI->second.getFlags();
2768 return true;
2769 });
2770 }
2771
2772 // Remove any weakly referenced symbols that haven't been resolved.
2773 IPLS->LookupSet.remove_if(
2774 Pred: [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2775 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2776 });
2777
2778 if (!IPLS->LookupSet.empty()) {
2779 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2780 return make_error<SymbolsNotFound>(Args: getSymbolStringPool(),
2781 Args: IPLS->LookupSet.getSymbolNames());
2782 }
2783
2784 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2785 return Result;
2786 });
2787
2788 // Run the callback on the result.
2789 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2790 OnComplete(std::move(Result));
2791}
2792
2793void ExecutionSession::OL_destroyMaterializationResponsibility(
2794 MaterializationResponsibility &MR) {
2795
2796 assert(MR.SymbolFlags.empty() &&
2797 "All symbols should have been explicitly materialized or failed");
2798 MR.JD.unlinkMaterializationResponsibility(MR);
2799}
2800
2801SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2802 const MaterializationResponsibility &MR) {
2803 return MR.JD.getRequestedSymbols(SymbolFlags: MR.SymbolFlags);
2804}
2805
2806Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2807 const SymbolMap &Symbols) {
2808 LLVM_DEBUG({
2809 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2810 });
2811#ifndef NDEBUG
2812 for (auto &KV : Symbols) {
2813 auto I = MR.SymbolFlags.find(KV.first);
2814 assert(I != MR.SymbolFlags.end() &&
2815 "Resolving symbol outside this responsibility set");
2816 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2817 "Can't resolve materialization-side-effects-only symbol");
2818 if (I->second & JITSymbolFlags::Common) {
2819 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2820 assert((KV.second.getFlags() & WeakOrCommon) &&
2821 "Common symbols must be resolved as common or weak");
2822 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2823 (I->second & ~JITSymbolFlags::Common) &&
2824 "Resolving symbol with incorrect flags");
2825 } else
2826 assert(KV.second.getFlags() == I->second &&
2827 "Resolving symbol with incorrect flags");
2828 }
2829#endif
2830
2831 return MR.JD.resolve(MR, Resolved: Symbols);
2832}
2833
2834WaitingOnGraph::ExternalState
2835ExecutionSession::IL_getSymbolState(JITDylib *JD,
2836 NonOwningSymbolStringPtr Name) {
2837 if (JD->State != JITDylib::Open)
2838 return WaitingOnGraph::ExternalState::Failed;
2839
2840 auto I = JD->Symbols.find_as(Val: Name);
2841
2842 // FIXME: Can we eliminate this possibility if we support query binding?
2843 if (I == JD->Symbols.end())
2844 return WaitingOnGraph::ExternalState::Failed;
2845
2846 if (I->second.getFlags().hasError())
2847 return WaitingOnGraph::ExternalState::Failed;
2848
2849 if (I->second.getState() == SymbolState::Ready)
2850 return WaitingOnGraph::ExternalState::Ready;
2851
2852 return WaitingOnGraph::ExternalState::None;
2853}
2854
2855template <typename UpdateSymbolFn, typename UpdateQueryFn>
2856void ExecutionSession::IL_collectQueries(
2857 JITDylib::AsynchronousSymbolQuerySet &Qs,
2858 WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
2859 UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
2860
2861 for (auto &[JD, Symbols] : QualifiedSymbols) {
2862 // IL_emit and JITDylib removal are synchronized by the session lock.
2863 // Since JITDylib removal removes any contained nodes from the
2864 // WaitingOnGraph, we should be able to assert that all nodes in the
2865 // WaitingOnGraph have not been removed.
2866 assert(JD->State == JITDylib::Open &&
2867 "WaitingOnGraph includes definition in defunct JITDylib");
2868 for (auto &Symbol : Symbols) {
2869 // Update symbol table.
2870 auto I = JD->Symbols.find_as(Val: Symbol);
2871 assert(I != JD->Symbols.end() &&
2872 "Failed Symbol missing from JD symbol table");
2873 auto &Entry = I->second;
2874 UpdateSymbol(Entry);
2875
2876 // Collect queries.
2877 auto J = JD->MaterializingInfos.find_as(Val: Symbol);
2878 if (J != JD->MaterializingInfos.end()) {
2879 for (auto &Q : J->second.takeAllPendingQueries()) {
2880 UpdateQuery(*Q, *JD, Symbol, Entry);
2881 Qs.insert(x: std::move(Q));
2882 }
2883 JD->MaterializingInfos.erase(I: J);
2884 }
2885 }
2886 }
2887}
2888
2889Expected<ExecutionSession::EmitQueries>
2890ExecutionSession::IL_emit(MaterializationResponsibility &MR,
2891 WaitingOnGraph::SimplifyResult SR) {
2892
2893 if (MR.RT->isDefunct())
2894 return make_error<ResourceTrackerDefunct>(Args&: MR.RT);
2895
2896 auto &TargetJD = MR.getTargetJITDylib();
2897 if (TargetJD.State != JITDylib::Open)
2898 return make_error<StringError>(Args: "JITDylib " + TargetJD.getName() +
2899 " is defunct",
2900 Args: inconvertibleErrorCode());
2901
2902#ifdef EXPENSIVE_CHECKS
2903 verifySessionState("entering ExecutionSession::IL_emit");
2904#endif
2905
2906 auto ER = G.emit(SR: std::move(SR),
2907 GetExternalState: [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
2908 return IL_getSymbolState(JD, Name);
2909 });
2910
2911 EmitQueries EQ;
2912
2913 // Handle failed queries.
2914 for (auto &SN : ER.Failed)
2915 IL_collectQueries(
2916 Qs&: EQ.Failed, QualifiedSymbols&: SN->defs(),
2917 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) {
2918 E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
2919 },
2920 UpdateQuery: [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
2921 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2922 auto &FS = EQ.FailedSymsForQuery[&Q];
2923 if (!FS)
2924 FS = std::make_shared<SymbolDependenceMap>();
2925 (*FS)[&JD].insert(V: SymbolStringPtr(Name));
2926 });
2927
2928 for (auto &FQ : EQ.Failed)
2929 FQ->detach();
2930
2931 for (auto &SN : ER.Ready)
2932 IL_collectQueries(
2933 Qs&: EQ.Completed, QualifiedSymbols&: SN->defs(),
2934 UpdateSymbol: [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
2935 UpdateQuery: [](AsynchronousSymbolQuery &Q, JITDylib &JD,
2936 NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
2937 Q.notifySymbolMetRequiredState(Name: SymbolStringPtr(Name), Sym: E.getSymbol());
2938 });
2939
2940 // std::erase_if is not available in C++17, and llvm::erase_if does not work
2941 // here.
2942 for (auto it = EQ.Completed.begin(), end = EQ.Completed.end(); it != end;) {
2943 if ((*it)->isComplete()) {
2944 ++it;
2945 } else {
2946 it = EQ.Completed.erase(position: it);
2947 }
2948 }
2949
2950#ifdef EXPENSIVE_CHECKS
2951 verifySessionState("exiting ExecutionSession::IL_emit");
2952#endif
2953
2954 return std::move(EQ);
2955}
2956
2957Error ExecutionSession::OL_notifyEmitted(
2958 MaterializationResponsibility &MR,
2959 ArrayRef<SymbolDependenceGroup> DepGroups) {
2960 LLVM_DEBUG({
2961 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
2962 << "\n";
2963 if (!DepGroups.empty()) {
2964 dbgs() << " Initial dependencies:\n";
2965 for (auto &SDG : DepGroups) {
2966 dbgs() << " Symbols: " << SDG.Symbols
2967 << ", Dependencies: " << SDG.Dependencies << "\n";
2968 }
2969 }
2970 });
2971
2972#ifndef NDEBUG
2973 SymbolNameSet Visited;
2974 for (auto &DG : DepGroups) {
2975 for (auto &Sym : DG.Symbols) {
2976 assert(MR.SymbolFlags.count(Sym) &&
2977 "DG contains dependence for symbol outside this MR");
2978 assert(Visited.insert(Sym).second &&
2979 "DG contains duplicate entries for Name");
2980 }
2981 }
2982#endif // NDEBUG
2983
2984 std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
2985 WaitingOnGraph::ContainerElementsMap Residual;
2986 {
2987 auto &JDResidual = Residual[&MR.getTargetJITDylib()];
2988 for (auto &[Name, Flags] : MR.getSymbols())
2989 JDResidual.insert(V: NonOwningSymbolStringPtr(Name));
2990
2991 for (auto &SDG : DepGroups) {
2992 WaitingOnGraph::ContainerElementsMap Defs;
2993 assert(!SDG.Symbols.empty());
2994 auto &JDDefs = Defs[&MR.getTargetJITDylib()];
2995 for (auto &Def : SDG.Symbols) {
2996 JDDefs.insert(V: NonOwningSymbolStringPtr(Def));
2997 JDResidual.erase(V: NonOwningSymbolStringPtr(Def));
2998 }
2999 WaitingOnGraph::ContainerElementsMap Deps;
3000 if (!SDG.Dependencies.empty()) {
3001 for (auto &[JD, Syms] : SDG.Dependencies) {
3002 auto &JDDeps = Deps[JD];
3003 for (auto &Dep : Syms)
3004 JDDeps.insert(V: NonOwningSymbolStringPtr(Dep));
3005 }
3006 }
3007 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
3008 args: std::move(Defs), args: std::move(Deps)));
3009 }
3010 if (!JDResidual.empty())
3011 SNs.push_back(x: std::make_unique<WaitingOnGraph::SuperNode>(
3012 args: std::move(Residual), args: WaitingOnGraph::ContainerElementsMap()));
3013 }
3014
3015 auto SR = WaitingOnGraph::simplify(SNs: std::move(SNs), Rec: GOpRecorder);
3016
3017 LLVM_DEBUG({
3018 dbgs() << " Simplified dependencies:\n";
3019 for (auto &SN : SR.superNodes()) {
3020
3021 auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
3022 std::vector<JITDylib *> JDs;
3023 for (auto &[JD, _] : C)
3024 JDs.push_back(JD);
3025 llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
3026 return LHS->getName() < RHS->getName();
3027 });
3028 return JDs;
3029 };
3030
3031 auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
3032 std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
3033 llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
3034 const NonOwningSymbolStringPtr &RHS) {
3035 return *LHS < *RHS;
3036 });
3037 return Names;
3038 };
3039
3040 dbgs() << " Defs: {";
3041 for (auto *JD : SortedLibs(SN->defs())) {
3042 dbgs() << " (" << JD->getName() << ", [";
3043 for (auto &Sym : SortedNames(SN->defs()[JD]))
3044 dbgs() << " " << Sym;
3045 dbgs() << " ])";
3046 }
3047 dbgs() << " }, Deps: {";
3048 for (auto *JD : SortedLibs(SN->deps())) {
3049 dbgs() << " (" << JD->getName() << ", [";
3050 for (auto &Sym : SortedNames(SN->deps()[JD]))
3051 dbgs() << " " << Sym;
3052 dbgs() << " ])";
3053 }
3054 dbgs() << " }\n";
3055 }
3056 });
3057 auto EmitQueries =
3058 runSessionLocked(F: [&]() { return IL_emit(MR, SR: std::move(SR)); });
3059
3060 // On error bail out.
3061 if (!EmitQueries)
3062 return EmitQueries.takeError();
3063
3064 // Otherwise notify failed queries, and any updated queries that have been
3065 // completed.
3066
3067 // FIXME: Get rid of error return from notifyEmitted.
3068 SymbolDependenceMap BadDeps;
3069 {
3070 for (auto &FQ : EmitQueries->Failed) {
3071 FQ->detach();
3072 assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
3073 "Missing failed symbols for query");
3074 auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
3075 for (auto &[JD, Syms] : *FailedSyms) {
3076 auto &BadDepsForJD = BadDeps[JD];
3077 for (auto &Sym : Syms)
3078 BadDepsForJD.insert(V: Sym);
3079 }
3080 FQ->handleFailed(Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(),
3081 Args: std::move(FailedSyms)));
3082 }
3083 }
3084
3085 for (auto &UQ : EmitQueries->Completed)
3086 UQ->handleComplete(ES&: *this);
3087
3088 // If there are any bad dependencies then return an error.
3089 if (!BadDeps.empty()) {
3090 SymbolNameSet BadNames;
3091 // Note: The name set calculated here is bogus: it includes all symbols in
3092 // the MR, not just the ones that failed. We want to remove the error
3093 // return path from notifyEmitted anyway, so this is just a brief
3094 // placeholder to maintain (roughly) the current error behavior.
3095 for (auto &[Name, Flags] : MR.getSymbols())
3096 BadNames.insert(V: Name);
3097 MR.SymbolFlags.clear();
3098 return make_error<UnsatisfiedSymbolDependencies>(
3099 Args: getSymbolStringPool(), Args: &MR.getTargetJITDylib(), Args: std::move(BadNames),
3100 Args: std::move(BadDeps), Args: "dependencies removed or in error state");
3101 }
3102
3103 MR.SymbolFlags.clear();
3104 return Error::success();
3105}
3106
3107Error ExecutionSession::OL_defineMaterializing(
3108 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3109
3110 LLVM_DEBUG({
3111 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3112 << NewSymbolFlags << "\n";
3113 });
3114 if (auto AcceptedDefs =
3115 MR.JD.defineMaterializing(FromMR&: MR, SymbolFlags: std::move(NewSymbolFlags))) {
3116 // Add all newly accepted symbols to this responsibility object.
3117 for (auto &KV : *AcceptedDefs)
3118 MR.SymbolFlags.insert(KV);
3119 return Error::success();
3120 } else
3121 return AcceptedDefs.takeError();
3122}
3123
3124std::pair<JITDylib::AsynchronousSymbolQuerySet,
3125 std::shared_ptr<SymbolDependenceMap>>
3126ExecutionSession::IL_failSymbols(JITDylib &JD,
3127 const SymbolNameVector &SymbolsToFail) {
3128
3129#ifdef EXPENSIVE_CHECKS
3130 verifySessionState("entering ExecutionSession::IL_failSymbols");
3131#endif
3132
3133 // Early out in the easy case.
3134 if (SymbolsToFail.empty())
3135 return {};
3136
3137 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3138 auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
3139 auto I = FailJD->Symbols.find_as(Val: FailSym);
3140 assert(I != FailJD->Symbols.end());
3141 I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
3142 auto J = FailJD->MaterializingInfos.find_as(Val: FailSym);
3143 if (J != FailJD->MaterializingInfos.end()) {
3144 for (auto &Q : J->second.takeAllPendingQueries())
3145 FailedQueries.insert(x: std::move(Q));
3146 FailJD->MaterializingInfos.erase(I: J);
3147 }
3148 };
3149
3150 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3151
3152 {
3153 auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
3154 for (auto &Sym : SymbolsToFail) {
3155 FailedSymsForJD.insert(V: Sym);
3156 Fail(&JD, NonOwningSymbolStringPtr(Sym));
3157 }
3158 }
3159
3160 WaitingOnGraph::ContainerElementsMap ToFail;
3161 auto &JDToFail = ToFail[&JD];
3162 for (auto &Sym : SymbolsToFail)
3163 JDToFail.insert(V: NonOwningSymbolStringPtr(Sym));
3164
3165 auto FailedSNs = G.fail(Failed: ToFail, Rec: GOpRecorder);
3166
3167 for (auto &SN : FailedSNs) {
3168 for (auto &[FailJD, Defs] : SN->defs()) {
3169 auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
3170 for (auto &Def : Defs) {
3171 FailedSymsForFailJD.insert(V: SymbolStringPtr(Def));
3172 Fail(FailJD, Def);
3173 }
3174 }
3175 }
3176
3177 // Detach all failed queries.
3178 for (auto &Q : FailedQueries)
3179 Q->detach();
3180
3181#ifdef EXPENSIVE_CHECKS
3182 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3183#endif
3184
3185 return std::make_pair(x: std::move(FailedQueries), y: std::move(FailedSymbolsMap));
3186}
3187
3188void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3189
3190 LLVM_DEBUG({
3191 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3192 << MR.SymbolFlags << "\n";
3193 });
3194
3195 if (MR.SymbolFlags.empty())
3196 return;
3197
3198 SymbolNameVector SymbolsToFail;
3199 for (auto &[Name, Flags] : MR.SymbolFlags)
3200 SymbolsToFail.push_back(x: Name);
3201 MR.SymbolFlags.clear();
3202
3203 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3204 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3205
3206 std::tie(args&: FailedQueries, args&: FailedSymbols) = runSessionLocked(F: [&]() {
3207 // If the tracker is defunct then there's nothing to do here.
3208 if (MR.RT->isDefunct())
3209 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3210 std::shared_ptr<SymbolDependenceMap>>();
3211 return IL_failSymbols(JD&: MR.getTargetJITDylib(), SymbolsToFail);
3212 });
3213
3214 for (auto &Q : FailedQueries) {
3215 Q->detach();
3216 Q->handleFailed(
3217 Err: make_error<FailedToMaterialize>(Args: getSymbolStringPool(), Args&: FailedSymbols));
3218 }
3219}
3220
3221Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3222 std::unique_ptr<MaterializationUnit> MU) {
3223 for (auto &KV : MU->getSymbols()) {
3224 assert(MR.SymbolFlags.count(KV.first) &&
3225 "Replacing definition outside this responsibility set");
3226 MR.SymbolFlags.erase(Val: KV.first);
3227 }
3228
3229 if (MU->getInitializerSymbol() == MR.InitSymbol)
3230 MR.InitSymbol = nullptr;
3231
3232 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3233 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3234 << "\n";
3235 }););
3236
3237 return MR.JD.replace(FromMR&: MR, MU: std::move(MU));
3238}
3239
3240Expected<std::unique_ptr<MaterializationResponsibility>>
3241ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3242 const SymbolNameSet &Symbols) {
3243
3244 SymbolStringPtr DelegatedInitSymbol;
3245 SymbolFlagsMap DelegatedFlags;
3246
3247 for (auto &Name : Symbols) {
3248 auto I = MR.SymbolFlags.find(Val: Name);
3249 assert(I != MR.SymbolFlags.end() &&
3250 "Symbol is not tracked by this MaterializationResponsibility "
3251 "instance");
3252
3253 DelegatedFlags[Name] = std::move(I->second);
3254 if (Name == MR.InitSymbol)
3255 std::swap(a&: MR.InitSymbol, b&: DelegatedInitSymbol);
3256
3257 MR.SymbolFlags.erase(I);
3258 }
3259
3260 return MR.JD.delegate(FromMR&: MR, SymbolFlags: std::move(DelegatedFlags),
3261 InitSymbol: std::move(DelegatedInitSymbol));
3262}
3263
3264#ifndef NDEBUG
3265void ExecutionSession::dumpDispatchInfo(Task &T) {
3266 runSessionLocked([&]() {
3267 dbgs() << "Dispatching: ";
3268 T.printDescription(dbgs());
3269 dbgs() << "\n";
3270 });
3271}
3272#endif // NDEBUG
3273
3274} // End namespace orc.
3275} // End namespace llvm.
3276