1//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10
11#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12#include "llvm/ExecutionEngine/Orc/MemoryAccess.h"
13#include "llvm/Support/MathExtras.h"
14
15#include <future>
16
17using namespace llvm;
18using namespace llvm::orc;
19
20namespace llvm {
21namespace orc {
22
23class EPCIndirectionUtilsAccess {
24public:
25 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
26 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
27
28 static Expected<IndirectStubInfoVector>
29 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
30 return EPCIU.getIndirectStubs(NumStubs);
31 };
32};
33
34} // end namespace orc
35} // end namespace llvm
36
37namespace {
38
39class EPCTrampolinePool : public TrampolinePool {
40public:
41 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
42 Error deallocatePool();
43
44protected:
45 Error grow() override;
46
47 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
48
49 EPCIndirectionUtils &EPCIU;
50 unsigned TrampolineSize = 0;
51 unsigned TrampolinesPerPage = 0;
52 std::vector<FinalizedAlloc> TrampolineBlocks;
53};
54
55class EPCIndirectStubsManager : public IndirectStubsManager,
56 private EPCIndirectionUtilsAccess {
57public:
58 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
59
60 Error createStub(StringRef StubName, ExecutorAddr StubAddr,
61 JITSymbolFlags StubFlags) override;
62
63 Error createStubs(const StubInitsMap &StubInits) override;
64
65 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
66
67 ExecutorSymbolDef findPointer(StringRef Name) override;
68
69 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
70
71private:
72 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
73
74 std::mutex ISMMutex;
75 EPCIndirectionUtils &EPCIU;
76 StringMap<StubInfo> StubInfos;
77};
78
79EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
80 : EPCIU(EPCIU) {
81 auto &EPC = EPCIU.getExecutorProcessControl();
82 auto &ABI = EPCIU.getABISupport();
83
84 TrampolineSize = ABI.getTrampolineSize();
85 TrampolinesPerPage =
86 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
87}
88
89Error EPCTrampolinePool::deallocatePool() {
90 std::promise<MSVCPError> DeallocResultP;
91 auto DeallocResultF = DeallocResultP.get_future();
92
93 EPCIU.getMemManager().deallocate(Allocs: std::move(TrampolineBlocks), OnDeallocated: [&](Error Err) {
94 DeallocResultP.set_value(std::move(Err));
95 });
96
97 return DeallocResultF.get();
98}
99
100Error EPCTrampolinePool::grow() {
101 using namespace jitlink;
102
103 assert(AvailableTrampolines.empty() &&
104 "Grow called with trampolines still available");
105
106 auto ResolverAddress = EPCIU.getResolverBlockAddress();
107 assert(ResolverAddress && "Resolver address can not be null");
108
109 auto &EPC = EPCIU.getExecutorProcessControl();
110 auto PageSize = EPC.getPageSize();
111 auto Alloc = SimpleSegmentAlloc::Create(
112 MemMgr&: EPCIU.getMemManager(), SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(),
113 JD: nullptr, Segments: {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
114 if (!Alloc)
115 return Alloc.takeError();
116
117 unsigned NumTrampolines = TrampolinesPerPage;
118
119 auto SegInfo = Alloc->getSegInfo(AG: MemProt::Read | MemProt::Exec);
120 EPCIU.getABISupport().writeTrampolines(
121 TrampolineBlockWorkingMem: SegInfo.WorkingMem.data(), TrampolineBlockTragetAddr: SegInfo.Addr, ResolverAddr: ResolverAddress, NumTrampolines);
122 for (unsigned I = 0; I < NumTrampolines; ++I)
123 AvailableTrampolines.push_back(x: SegInfo.Addr + (I * TrampolineSize));
124
125 auto FA = Alloc->finalize();
126 if (!FA)
127 return FA.takeError();
128
129 TrampolineBlocks.push_back(x: std::move(*FA));
130
131 return Error::success();
132}
133
134Error EPCIndirectStubsManager::createStub(StringRef StubName,
135 ExecutorAddr StubAddr,
136 JITSymbolFlags StubFlags) {
137 StubInitsMap SIM;
138 SIM[StubName] = std::make_pair(x&: StubAddr, y&: StubFlags);
139 return createStubs(StubInits: SIM);
140}
141
142Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
143 auto AvailableStubInfos = getIndirectStubs(EPCIU, NumStubs: StubInits.size());
144 if (!AvailableStubInfos)
145 return AvailableStubInfos.takeError();
146
147 {
148 std::lock_guard<std::mutex> Lock(ISMMutex);
149 unsigned ASIdx = 0;
150 for (auto &SI : StubInits) {
151 auto &A = (*AvailableStubInfos)[ASIdx++];
152 StubInfos[SI.first()] = std::make_pair(x&: A, y: SI.second.second);
153 }
154 }
155
156 auto &MemAccess = EPCIU.getMemoryAccess();
157 switch (EPCIU.getABISupport().getPointerSize()) {
158 case 4: {
159 unsigned ASIdx = 0;
160 std::vector<tpctypes::UInt32Write> PtrUpdates;
161 for (auto &SI : StubInits)
162 PtrUpdates.push_back(x: {(*AvailableStubInfos)[ASIdx++].PointerAddress,
163 static_cast<uint32_t>(SI.second.first.getValue())});
164 return MemAccess.writeUInt32s(Ws: PtrUpdates);
165 }
166 case 8: {
167 unsigned ASIdx = 0;
168 std::vector<tpctypes::UInt64Write> PtrUpdates;
169 for (auto &SI : StubInits)
170 PtrUpdates.push_back(x: {(*AvailableStubInfos)[ASIdx++].PointerAddress,
171 SI.second.first.getValue()});
172 return MemAccess.writeUInt64s(Ws: PtrUpdates);
173 }
174 default:
175 return make_error<StringError>(Args: "Unsupported pointer size",
176 Args: inconvertibleErrorCode());
177 }
178}
179
180ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
181 bool ExportedStubsOnly) {
182 std::lock_guard<std::mutex> Lock(ISMMutex);
183 auto I = StubInfos.find(Key: Name);
184 if (I == StubInfos.end())
185 return ExecutorSymbolDef();
186 return {I->second.first.StubAddress, I->second.second};
187}
188
189ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
190 std::lock_guard<std::mutex> Lock(ISMMutex);
191 auto I = StubInfos.find(Key: Name);
192 if (I == StubInfos.end())
193 return ExecutorSymbolDef();
194 return {I->second.first.PointerAddress, I->second.second};
195}
196
197Error EPCIndirectStubsManager::updatePointer(StringRef Name,
198 ExecutorAddr NewAddr) {
199
200 ExecutorAddr PtrAddr;
201 {
202 std::lock_guard<std::mutex> Lock(ISMMutex);
203 auto I = StubInfos.find(Key: Name);
204 if (I == StubInfos.end())
205 return make_error<StringError>(Args: "Unknown stub name",
206 Args: inconvertibleErrorCode());
207 PtrAddr = I->second.first.PointerAddress;
208 }
209
210 auto &MemAccess = EPCIU.getMemoryAccess();
211 switch (EPCIU.getABISupport().getPointerSize()) {
212 case 4: {
213 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
214 return MemAccess.writeUInt32s(Ws: PUpdate);
215 }
216 case 8: {
217 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
218 return MemAccess.writeUInt64s(Ws: PUpdate);
219 }
220 default:
221 return make_error<StringError>(Args: "Unsupported pointer size",
222 Args: inconvertibleErrorCode());
223 }
224}
225
226} // end anonymous namespace.
227
228namespace llvm {
229namespace orc {
230
231EPCIndirectionUtils::ABISupport::~ABISupport() = default;
232
233Expected<std::unique_ptr<EPCIndirectionUtils>>
234EPCIndirectionUtils::Create(ExecutorProcessControl &EPC,
235 jitlink::JITLinkMemoryManager &MemMgr,
236 MemoryAccess &MemAccess) {
237 const auto &TT = EPC.getTargetTriple();
238 switch (TT.getArch()) {
239 default:
240 return make_error<StringError>(
241 Args: std::string("No EPCIndirectionUtils available for ") + TT.str(),
242 Args: inconvertibleErrorCode());
243 case Triple::aarch64:
244 case Triple::aarch64_32:
245 return CreateWithABI<OrcAArch64>(EPC, MemMgr, MemAccess);
246
247 case Triple::x86:
248 return CreateWithABI<OrcI386>(EPC, MemMgr, MemAccess);
249
250 case Triple::loongarch64:
251 return CreateWithABI<OrcLoongArch64>(EPC, MemMgr, MemAccess);
252
253 case Triple::mips:
254 return CreateWithABI<OrcMips32Be>(EPC, MemMgr, MemAccess);
255
256 case Triple::mipsel:
257 return CreateWithABI<OrcMips32Le>(EPC, MemMgr, MemAccess);
258
259 case Triple::mips64:
260 case Triple::mips64el:
261 return CreateWithABI<OrcMips64>(EPC, MemMgr, MemAccess);
262
263 case Triple::riscv64:
264 return CreateWithABI<OrcRiscv64>(EPC, MemMgr, MemAccess);
265
266 case Triple::x86_64:
267 if (TT.getOS() == Triple::OSType::Win32)
268 return CreateWithABI<OrcX86_64_Win32>(EPC, MemMgr, MemAccess);
269 else
270 return CreateWithABI<OrcX86_64_SysV>(EPC, MemMgr, MemAccess);
271 }
272}
273
274Error EPCIndirectionUtils::cleanup() {
275
276 auto Err = MemMgr.deallocate(Allocs: std::move(IndirectStubAllocs));
277
278 if (TP)
279 Err = joinErrors(E1: std::move(Err),
280 E2: static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
281
282 if (ResolverBlock)
283 Err =
284 joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(ResolverBlock)));
285
286 return Err;
287}
288
289Expected<ExecutorAddr>
290EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
291 ExecutorAddr ReentryCtxAddr) {
292 using namespace jitlink;
293
294 assert(ABI && "ABI can not be null");
295 auto ResolverSize = ABI->getResolverCodeSize();
296
297 auto Alloc = SimpleSegmentAlloc::Create(
298 MemMgr, SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(), JD: nullptr,
299 Segments: {{MemProt::Read | MemProt::Exec,
300 {ResolverSize, Align(EPC.getPageSize())}}});
301
302 if (!Alloc)
303 return Alloc.takeError();
304
305 auto SegInfo = Alloc->getSegInfo(AG: MemProt::Read | MemProt::Exec);
306 ResolverBlockAddr = SegInfo.Addr;
307 ABI->writeResolverCode(ResolverWorkingMem: SegInfo.WorkingMem.data(), ResolverTargetAddr: ResolverBlockAddr,
308 ReentryFnAddr, ReentryCtxAddr);
309
310 auto FA = Alloc->finalize();
311 if (!FA)
312 return FA.takeError();
313
314 ResolverBlock = std::move(*FA);
315 return ResolverBlockAddr;
316}
317
318std::unique_ptr<IndirectStubsManager>
319EPCIndirectionUtils::createIndirectStubsManager() {
320 return std::make_unique<EPCIndirectStubsManager>(args&: *this);
321}
322
323TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
324 if (!TP)
325 TP = std::make_unique<EPCTrampolinePool>(args&: *this);
326 return *TP;
327}
328
329LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
330 ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
331 assert(!LCTM &&
332 "createLazyCallThroughManager can not have been called before");
333 LCTM = std::make_unique<LazyCallThroughManager>(args&: ES, args&: ErrorHandlerAddr,
334 args: &getTrampolinePool());
335 return *LCTM;
336}
337
338EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
339 jitlink::JITLinkMemoryManager &MemMgr,
340 MemoryAccess &MemAccess,
341 std::unique_ptr<ABISupport> ABI)
342 : EPC(EPC), MemMgr(MemMgr), MemAccess(MemAccess), ABI(std::move(ABI)) {
343 assert(this->ABI && "ABI can not be null");
344
345 assert(EPC.getPageSize() > getABISupport().getStubSize() &&
346 "Stubs larger than one page are not supported");
347}
348
349Expected<EPCIndirectionUtils::IndirectStubInfoVector>
350EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
351 using namespace jitlink;
352
353 std::lock_guard<std::mutex> Lock(EPCUIMutex);
354
355 // If there aren't enough stubs available then allocate some more.
356 if (NumStubs > AvailableIndirectStubs.size()) {
357 auto NumStubsToAllocate = NumStubs;
358 auto PageSize = EPC.getPageSize();
359 auto StubBytes = alignTo(Value: NumStubsToAllocate * ABI->getStubSize(), Align: PageSize);
360 NumStubsToAllocate = StubBytes / ABI->getStubSize();
361 auto PtrBytes =
362 alignTo(Value: NumStubsToAllocate * ABI->getPointerSize(), Align: PageSize);
363
364 auto StubProt = MemProt::Read | MemProt::Exec;
365 auto PtrProt = MemProt::Read | MemProt::Write;
366
367 auto Alloc = SimpleSegmentAlloc::Create(
368 MemMgr, SSP: EPC.getSymbolStringPool(), TT: EPC.getTargetTriple(), JD: nullptr,
369 Segments: {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
370 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
371
372 if (!Alloc)
373 return Alloc.takeError();
374
375 auto StubSeg = Alloc->getSegInfo(AG: StubProt);
376 auto PtrSeg = Alloc->getSegInfo(AG: PtrProt);
377
378 ABI->writeIndirectStubsBlock(StubsBlockWorkingMem: StubSeg.WorkingMem.data(), StubsBlockTargetAddress: StubSeg.Addr,
379 PointersBlockTargetAddress: PtrSeg.Addr, NumStubs: NumStubsToAllocate);
380
381 auto FA = Alloc->finalize();
382 if (!FA)
383 return FA.takeError();
384
385 IndirectStubAllocs.push_back(x: std::move(*FA));
386
387 auto StubExecutorAddr = StubSeg.Addr;
388 auto PtrExecutorAddr = PtrSeg.Addr;
389 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
390 AvailableIndirectStubs.push_back(
391 x: IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
392 StubExecutorAddr += ABI->getStubSize();
393 PtrExecutorAddr += ABI->getPointerSize();
394 }
395 }
396
397 assert(NumStubs <= AvailableIndirectStubs.size() &&
398 "Sufficient stubs should have been allocated above");
399
400 IndirectStubInfoVector Result;
401 while (NumStubs--) {
402 Result.push_back(x: AvailableIndirectStubs.back());
403 AvailableIndirectStubs.pop_back();
404 }
405
406 return std::move(Result);
407}
408
409static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
410 JITTargetAddress TrampolineAddr) {
411 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(Addr: LCTMAddr);
412 std::promise<ExecutorAddr> LandingAddrP;
413 auto LandingAddrF = LandingAddrP.get_future();
414 LCTM.resolveTrampolineLandingAddress(
415 TrampolineAddr: ExecutorAddr(TrampolineAddr),
416 NotifyLandingResolved: [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
417 return LandingAddrF.get().getValue();
418}
419
420Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
421 auto &LCTM = EPCIU.getLazyCallThroughManager();
422 return EPCIU
423 .writeResolverBlock(ReentryFnAddr: ExecutorAddr::fromPtr(Ptr: &reentry),
424 ReentryCtxAddr: ExecutorAddr::fromPtr(Ptr: &LCTM))
425 .takeError();
426}
427
428} // end namespace orc
429} // end namespace llvm
430