1//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10
11#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12#include "llvm/ExecutionEngine/Orc/MemoryAccess.h"
13#include "llvm/Support/MathExtras.h"
14
15#include <future>
16
17using namespace llvm;
18using namespace llvm::orc;
19
20namespace llvm {
21namespace orc {
22
23class EPCIndirectionUtilsAccess {
24public:
25 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
26 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
27
28 static Expected<IndirectStubInfoVector>
29 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
30 return EPCIU.getIndirectStubs(NumStubs);
31 };
32};
33
34} // end namespace orc
35} // end namespace llvm
36
37namespace {
38
39class EPCTrampolinePool : public TrampolinePool {
40public:
41 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
42 Error deallocatePool();
43
44protected:
45 Error grow() override;
46
47 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
48
49 EPCIndirectionUtils &EPCIU;
50 unsigned TrampolineSize = 0;
51 unsigned TrampolinesPerPage = 0;
52 std::vector<FinalizedAlloc> TrampolineBlocks;
53};
54
55class EPCIndirectStubsManager : public IndirectStubsManager,
56 private EPCIndirectionUtilsAccess {
57public:
58 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
59
60 Error deallocateStubs();
61
62 Error createStub(StringRef StubName, ExecutorAddr StubAddr,
63 JITSymbolFlags StubFlags) override;
64
65 Error createStubs(const StubInitsMap &StubInits) override;
66
67 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
68
69 ExecutorSymbolDef findPointer(StringRef Name) override;
70
71 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
72
73private:
74 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
75
76 std::mutex ISMMutex;
77 EPCIndirectionUtils &EPCIU;
78 StringMap<StubInfo> StubInfos;
79};
80
81EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
82 : EPCIU(EPCIU) {
83 auto &EPC = EPCIU.getExecutorProcessControl();
84 auto &ABI = EPCIU.getABISupport();
85
86 TrampolineSize = ABI.getTrampolineSize();
87 TrampolinesPerPage =
88 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
89}
90
91Error EPCTrampolinePool::deallocatePool() {
92 std::promise<MSVCPError> DeallocResultP;
93 auto DeallocResultF = DeallocResultP.get_future();
94
95 EPCIU.getMemManager().deallocate(Allocs: std::move(TrampolineBlocks), OnDeallocated: [&](Error Err) {
96 DeallocResultP.set_value(std::move(Err));
97 });
98
99 return DeallocResultF.get();
100}
101
102Error EPCTrampolinePool::grow() {
103 using namespace jitlink;
104
105 assert(AvailableTrampolines.empty() &&
106 "Grow called with trampolines still available");
107
108 auto ResolverAddress = EPCIU.getResolverBlockAddress();
109 assert(ResolverAddress && "Resolver address can not be null");
110
111 auto &EPC = EPCIU.getExecutorProcessControl();
112 auto PageSize = EPC.getPageSize();
113 auto Alloc = SimpleSegmentAlloc::Create(
114 MemMgr&: EPCIU.getMemManager(), SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(),
115 JD: nullptr, Segments: {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
116 if (!Alloc)
117 return Alloc.takeError();
118
119 unsigned NumTrampolines = TrampolinesPerPage;
120
121 auto SegInfo = Alloc->getSegInfo(AG: MemProt::Read | MemProt::Exec);
122 EPCIU.getABISupport().writeTrampolines(
123 TrampolineBlockWorkingMem: SegInfo.WorkingMem.data(), TrampolineBlockTragetAddr: SegInfo.Addr, ResolverAddr: ResolverAddress, NumTrampolines);
124 for (unsigned I = 0; I < NumTrampolines; ++I)
125 AvailableTrampolines.push_back(x: SegInfo.Addr + (I * TrampolineSize));
126
127 auto FA = Alloc->finalize();
128 if (!FA)
129 return FA.takeError();
130
131 TrampolineBlocks.push_back(x: std::move(*FA));
132
133 return Error::success();
134}
135
136Error EPCIndirectStubsManager::createStub(StringRef StubName,
137 ExecutorAddr StubAddr,
138 JITSymbolFlags StubFlags) {
139 StubInitsMap SIM;
140 SIM[StubName] = std::make_pair(x&: StubAddr, y&: StubFlags);
141 return createStubs(StubInits: SIM);
142}
143
144Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
145 auto AvailableStubInfos = getIndirectStubs(EPCIU, NumStubs: StubInits.size());
146 if (!AvailableStubInfos)
147 return AvailableStubInfos.takeError();
148
149 {
150 std::lock_guard<std::mutex> Lock(ISMMutex);
151 unsigned ASIdx = 0;
152 for (auto &SI : StubInits) {
153 auto &A = (*AvailableStubInfos)[ASIdx++];
154 StubInfos[SI.first()] = std::make_pair(x&: A, y: SI.second.second);
155 }
156 }
157
158 auto &MemAccess = EPCIU.getMemoryAccess();
159 switch (EPCIU.getABISupport().getPointerSize()) {
160 case 4: {
161 unsigned ASIdx = 0;
162 std::vector<tpctypes::UInt32Write> PtrUpdates;
163 for (auto &SI : StubInits)
164 PtrUpdates.push_back(x: {(*AvailableStubInfos)[ASIdx++].PointerAddress,
165 static_cast<uint32_t>(SI.second.first.getValue())});
166 return MemAccess.writeUInt32s(Ws: PtrUpdates);
167 }
168 case 8: {
169 unsigned ASIdx = 0;
170 std::vector<tpctypes::UInt64Write> PtrUpdates;
171 for (auto &SI : StubInits)
172 PtrUpdates.push_back(x: {(*AvailableStubInfos)[ASIdx++].PointerAddress,
173 SI.second.first.getValue()});
174 return MemAccess.writeUInt64s(Ws: PtrUpdates);
175 }
176 default:
177 return make_error<StringError>(Args: "Unsupported pointer size",
178 Args: inconvertibleErrorCode());
179 }
180}
181
182ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
183 bool ExportedStubsOnly) {
184 std::lock_guard<std::mutex> Lock(ISMMutex);
185 auto I = StubInfos.find(Key: Name);
186 if (I == StubInfos.end())
187 return ExecutorSymbolDef();
188 return {I->second.first.StubAddress, I->second.second};
189}
190
191ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
192 std::lock_guard<std::mutex> Lock(ISMMutex);
193 auto I = StubInfos.find(Key: Name);
194 if (I == StubInfos.end())
195 return ExecutorSymbolDef();
196 return {I->second.first.PointerAddress, I->second.second};
197}
198
199Error EPCIndirectStubsManager::updatePointer(StringRef Name,
200 ExecutorAddr NewAddr) {
201
202 ExecutorAddr PtrAddr;
203 {
204 std::lock_guard<std::mutex> Lock(ISMMutex);
205 auto I = StubInfos.find(Key: Name);
206 if (I == StubInfos.end())
207 return make_error<StringError>(Args: "Unknown stub name",
208 Args: inconvertibleErrorCode());
209 PtrAddr = I->second.first.PointerAddress;
210 }
211
212 auto &MemAccess = EPCIU.getMemoryAccess();
213 switch (EPCIU.getABISupport().getPointerSize()) {
214 case 4: {
215 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
216 return MemAccess.writeUInt32s(Ws: PUpdate);
217 }
218 case 8: {
219 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
220 return MemAccess.writeUInt64s(Ws: PUpdate);
221 }
222 default:
223 return make_error<StringError>(Args: "Unsupported pointer size",
224 Args: inconvertibleErrorCode());
225 }
226}
227
228} // end anonymous namespace.
229
230namespace llvm {
231namespace orc {
232
233EPCIndirectionUtils::ABISupport::~ABISupport() = default;
234
235Expected<std::unique_ptr<EPCIndirectionUtils>>
236EPCIndirectionUtils::Create(ExecutorProcessControl &EPC,
237 jitlink::JITLinkMemoryManager &MemMgr,
238 MemoryAccess &MemAccess) {
239 const auto &TT = EPC.getTargetTriple();
240 switch (TT.getArch()) {
241 default:
242 return make_error<StringError>(
243 Args: std::string("No EPCIndirectionUtils available for ") + TT.str(),
244 Args: inconvertibleErrorCode());
245 case Triple::aarch64:
246 case Triple::aarch64_32:
247 return CreateWithABI<OrcAArch64>(EPC, MemMgr, MemAccess);
248
249 case Triple::x86:
250 return CreateWithABI<OrcI386>(EPC, MemMgr, MemAccess);
251
252 case Triple::loongarch64:
253 return CreateWithABI<OrcLoongArch64>(EPC, MemMgr, MemAccess);
254
255 case Triple::mips:
256 return CreateWithABI<OrcMips32Be>(EPC, MemMgr, MemAccess);
257
258 case Triple::mipsel:
259 return CreateWithABI<OrcMips32Le>(EPC, MemMgr, MemAccess);
260
261 case Triple::mips64:
262 case Triple::mips64el:
263 return CreateWithABI<OrcMips64>(EPC, MemMgr, MemAccess);
264
265 case Triple::riscv64:
266 return CreateWithABI<OrcRiscv64>(EPC, MemMgr, MemAccess);
267
268 case Triple::x86_64:
269 if (TT.getOS() == Triple::OSType::Win32)
270 return CreateWithABI<OrcX86_64_Win32>(EPC, MemMgr, MemAccess);
271 else
272 return CreateWithABI<OrcX86_64_SysV>(EPC, MemMgr, MemAccess);
273 }
274}
275
276Error EPCIndirectionUtils::cleanup() {
277
278 auto Err = MemMgr.deallocate(Allocs: std::move(IndirectStubAllocs));
279
280 if (TP)
281 Err = joinErrors(E1: std::move(Err),
282 E2: static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
283
284 if (ResolverBlock)
285 Err =
286 joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(ResolverBlock)));
287
288 return Err;
289}
290
291Expected<ExecutorAddr>
292EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
293 ExecutorAddr ReentryCtxAddr) {
294 using namespace jitlink;
295
296 assert(ABI && "ABI can not be null");
297 auto ResolverSize = ABI->getResolverCodeSize();
298
299 auto Alloc = SimpleSegmentAlloc::Create(
300 MemMgr, SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(), JD: nullptr,
301 Segments: {{MemProt::Read | MemProt::Exec,
302 {ResolverSize, Align(EPC.getPageSize())}}});
303
304 if (!Alloc)
305 return Alloc.takeError();
306
307 auto SegInfo = Alloc->getSegInfo(AG: MemProt::Read | MemProt::Exec);
308 ResolverBlockAddr = SegInfo.Addr;
309 ABI->writeResolverCode(ResolverWorkingMem: SegInfo.WorkingMem.data(), ResolverTargetAddr: ResolverBlockAddr,
310 ReentryFnAddr, ReentryCtxAddr);
311
312 auto FA = Alloc->finalize();
313 if (!FA)
314 return FA.takeError();
315
316 ResolverBlock = std::move(*FA);
317 return ResolverBlockAddr;
318}
319
320std::unique_ptr<IndirectStubsManager>
321EPCIndirectionUtils::createIndirectStubsManager() {
322 return std::make_unique<EPCIndirectStubsManager>(args&: *this);
323}
324
325TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
326 if (!TP)
327 TP = std::make_unique<EPCTrampolinePool>(args&: *this);
328 return *TP;
329}
330
331LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
332 ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
333 assert(!LCTM &&
334 "createLazyCallThroughManager can not have been called before");
335 LCTM = std::make_unique<LazyCallThroughManager>(args&: ES, args&: ErrorHandlerAddr,
336 args: &getTrampolinePool());
337 return *LCTM;
338}
339
340EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
341 jitlink::JITLinkMemoryManager &MemMgr,
342 MemoryAccess &MemAccess,
343 std::unique_ptr<ABISupport> ABI)
344 : EPC(EPC), MemMgr(MemMgr), MemAccess(MemAccess), ABI(std::move(ABI)) {
345 assert(this->ABI && "ABI can not be null");
346
347 assert(EPC.getPageSize() > getABISupport().getStubSize() &&
348 "Stubs larger than one page are not supported");
349}
350
351Expected<EPCIndirectionUtils::IndirectStubInfoVector>
352EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
353 using namespace jitlink;
354
355 std::lock_guard<std::mutex> Lock(EPCUIMutex);
356
357 // If there aren't enough stubs available then allocate some more.
358 if (NumStubs > AvailableIndirectStubs.size()) {
359 auto NumStubsToAllocate = NumStubs;
360 auto PageSize = EPC.getPageSize();
361 auto StubBytes = alignTo(Value: NumStubsToAllocate * ABI->getStubSize(), Align: PageSize);
362 NumStubsToAllocate = StubBytes / ABI->getStubSize();
363 auto PtrBytes =
364 alignTo(Value: NumStubsToAllocate * ABI->getPointerSize(), Align: PageSize);
365
366 auto StubProt = MemProt::Read | MemProt::Exec;
367 auto PtrProt = MemProt::Read | MemProt::Write;
368
369 auto Alloc = SimpleSegmentAlloc::Create(
370 MemMgr, SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(), JD: nullptr,
371 Segments: {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
372 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
373
374 if (!Alloc)
375 return Alloc.takeError();
376
377 auto StubSeg = Alloc->getSegInfo(AG: StubProt);
378 auto PtrSeg = Alloc->getSegInfo(AG: PtrProt);
379
380 ABI->writeIndirectStubsBlock(StubsBlockWorkingMem: StubSeg.WorkingMem.data(), StubsBlockTargetAddress: StubSeg.Addr,
381 PointersBlockTargetAddress: PtrSeg.Addr, NumStubs: NumStubsToAllocate);
382
383 auto FA = Alloc->finalize();
384 if (!FA)
385 return FA.takeError();
386
387 IndirectStubAllocs.push_back(x: std::move(*FA));
388
389 auto StubExecutorAddr = StubSeg.Addr;
390 auto PtrExecutorAddr = PtrSeg.Addr;
391 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
392 AvailableIndirectStubs.push_back(
393 x: IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
394 StubExecutorAddr += ABI->getStubSize();
395 PtrExecutorAddr += ABI->getPointerSize();
396 }
397 }
398
399 assert(NumStubs <= AvailableIndirectStubs.size() &&
400 "Sufficient stubs should have been allocated above");
401
402 IndirectStubInfoVector Result;
403 while (NumStubs--) {
404 Result.push_back(x: AvailableIndirectStubs.back());
405 AvailableIndirectStubs.pop_back();
406 }
407
408 return std::move(Result);
409}
410
411static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
412 JITTargetAddress TrampolineAddr) {
413 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(Addr: LCTMAddr);
414 std::promise<ExecutorAddr> LandingAddrP;
415 auto LandingAddrF = LandingAddrP.get_future();
416 LCTM.resolveTrampolineLandingAddress(
417 TrampolineAddr: ExecutorAddr(TrampolineAddr),
418 NotifyLandingResolved: [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
419 return LandingAddrF.get().getValue();
420}
421
422Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
423 auto &LCTM = EPCIU.getLazyCallThroughManager();
424 return EPCIU
425 .writeResolverBlock(ReentryFnAddr: ExecutorAddr::fromPtr(Ptr: &reentry),
426 ReentryCtxAddr: ExecutorAddr::fromPtr(Ptr: &LCTM))
427 .takeError();
428}
429
430} // end namespace orc
431} // end namespace llvm
432