1//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
11#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12#include "llvm/Support/MSVCErrorWorkarounds.h"
13#include "llvm/Support/Process.h"
14#include "llvm/Support/WindowsError.h"
15#include <future>
16#include <sstream>
17
18#if defined(LLVM_ON_UNIX)
19#include <errno.h>
20#include <fcntl.h>
21#include <sys/mman.h>
22#if defined(__MVS__)
23#include "llvm/Support/BLAKE3.h"
24#include <sys/shm.h>
25#endif
26#include <unistd.h>
27#endif
28
29namespace llvm {
30namespace orc {
31namespace rt_bootstrap {
32
33#if defined(_WIN32)
34static DWORD getWindowsProtectionFlags(MemProt MP) {
35 if (MP == MemProt::Read)
36 return PAGE_READONLY;
37 if (MP == MemProt::Write ||
38 MP == (MemProt::Write | MemProt::Read)) {
39 // Note: PAGE_WRITE is not supported by VirtualProtect
40 return PAGE_READWRITE;
41 }
42 if (MP == (MemProt::Read | MemProt::Exec))
43 return PAGE_EXECUTE_READ;
44 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
45 return PAGE_EXECUTE_READWRITE;
46 if (MP == MemProt::Exec)
47 return PAGE_EXECUTE;
48
49 return PAGE_NOACCESS;
50}
51#endif
52
53Expected<std::pair<ExecutorAddr, std::string>>
54ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
55#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
56
57#if defined(LLVM_ON_UNIX)
58
59 std::string SharedMemoryName;
60 {
61 std::stringstream SharedMemoryNameStream;
62 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
63 << (++SharedMemoryCount);
64 SharedMemoryName = SharedMemoryNameStream.str();
65 }
66
67#if defined(__MVS__)
68 ArrayRef<uint8_t> Data(
69 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
70 SharedMemoryName.size());
71 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
72 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
73 int SharedMemoryId =
74 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
75 if (SharedMemoryId < 0)
76 return errorCodeToError(errnoAsErrorCode());
77
78 void *Addr = shmat(SharedMemoryId, nullptr, 0);
79 if (Addr == reinterpret_cast<void *>(-1))
80 return errorCodeToError(errnoAsErrorCode());
81#else
82 int SharedMemoryFile =
83 shm_open(name: SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, mode: 0700);
84 if (SharedMemoryFile < 0)
85 return errorCodeToError(EC: errnoAsErrorCode());
86
87 // by default size is 0
88 if (ftruncate(fd: SharedMemoryFile, length: Size) < 0)
89 return errorCodeToError(EC: errnoAsErrorCode());
90
91 void *Addr = mmap(addr: nullptr, len: Size, PROT_NONE, MAP_SHARED, fd: SharedMemoryFile, offset: 0);
92 if (Addr == MAP_FAILED)
93 return errorCodeToError(EC: errnoAsErrorCode());
94
95 close(fd: SharedMemoryFile);
96#endif
97
98#elif defined(_WIN32)
99
100 std::string SharedMemoryName;
101 {
102 std::stringstream SharedMemoryNameStream;
103 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
104 << (++SharedMemoryCount);
105 SharedMemoryName = SharedMemoryNameStream.str();
106 }
107
108 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
109 SharedMemoryName.end());
110 HANDLE SharedMemoryFile = CreateFileMappingW(
111 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
112 Size & 0xffffffff, WideSharedMemoryName.c_str());
113 if (!SharedMemoryFile)
114 return errorCodeToError(mapWindowsError(GetLastError()));
115
116 void *Addr = MapViewOfFile(SharedMemoryFile,
117 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
118 if (!Addr) {
119 CloseHandle(SharedMemoryFile);
120 return errorCodeToError(mapWindowsError(GetLastError()));
121 }
122
123#endif
124
125 {
126 std::lock_guard<std::mutex> Lock(Mutex);
127 Reservations[Addr].Size = Size;
128#if defined(_WIN32)
129 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
130#endif
131 }
132
133 return std::make_pair(x: ExecutorAddr::fromPtr(Ptr: Addr),
134 y: std::move(SharedMemoryName));
135#else
136 return make_error<StringError>(
137 "SharedMemoryMapper is not supported on this platform yet",
138 inconvertibleErrorCode());
139#endif
140}
141
142Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
143 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
144#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
145
146 ExecutorAddr MinAddr(~0ULL);
147
148 // Contents are already in place
149 for (auto &Segment : FR.Segments) {
150 if (Segment.Addr < MinAddr)
151 MinAddr = Segment.Addr;
152
153#if defined(LLVM_ON_UNIX)
154
155#if defined(__MVS__)
156 // TODO Is it possible to change the protection level?
157#else
158 int NativeProt = 0;
159 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
160 NativeProt |= PROT_READ;
161 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
162 NativeProt |= PROT_WRITE;
163 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
164 NativeProt |= PROT_EXEC;
165
166 if (mprotect(addr: Segment.Addr.toPtr<void *>(), len: Segment.Size, prot: NativeProt))
167 return errorCodeToError(EC: errnoAsErrorCode());
168#endif
169
170#elif defined(_WIN32)
171
172 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
173
174 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
175 &NativeProt))
176 return errorCodeToError(mapWindowsError(GetLastError()));
177
178#endif
179
180 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
181 sys::Memory::InvalidateInstructionCache(Addr: Segment.Addr.toPtr<void *>(),
182 Len: Segment.Size);
183 }
184
185 // Run finalization actions and get deinitlization action list.
186 std::vector<shared::WrapperFunctionCall> DeinitializeActions;
187 {
188 std::promise<MSVCPExpected<std::vector<shared::WrapperFunctionCall>>> P;
189 auto F = P.get_future();
190 shared::runFinalizeActions(
191 AAs&: FR.Actions, OnComplete: [&](Expected<std::vector<shared::WrapperFunctionCall>> R) {
192 P.set_value(std::move(R));
193 });
194 if (auto DeinitializeActionsOrErr = F.get())
195 DeinitializeActions = std::move(*DeinitializeActionsOrErr);
196 else
197 return DeinitializeActionsOrErr.takeError();
198 }
199
200 {
201 std::lock_guard<std::mutex> Lock(Mutex);
202 Allocations[MinAddr].DeinitializationActions =
203 std::move(DeinitializeActions);
204 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(x: MinAddr);
205 }
206
207 return MinAddr;
208
209#else
210 return make_error<StringError>(
211 "SharedMemoryMapper is not supported on this platform yet",
212 inconvertibleErrorCode());
213#endif
214}
215
216Error ExecutorSharedMemoryMapperService::deinitialize(
217 const std::vector<ExecutorAddr> &Bases) {
218 Error AllErr = Error::success();
219
220 {
221 std::lock_guard<std::mutex> Lock(Mutex);
222
223 for (auto Base : llvm::reverse(C: Bases)) {
224 shared::runDeallocActions(
225 DAs: Allocations[Base].DeinitializationActions, OnComplete: [&](Error Err) {
226 if (Err)
227 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
228 });
229
230 // Remove the allocation from the allocation list of its reservation
231 for (auto &Reservation : Reservations) {
232 auto AllocationIt = llvm::find(Range&: Reservation.second.Allocations, Val: Base);
233 if (AllocationIt != Reservation.second.Allocations.end()) {
234 Reservation.second.Allocations.erase(position: AllocationIt);
235 break;
236 }
237 }
238
239 Allocations.erase(Val: Base);
240 }
241 }
242
243 return AllErr;
244}
245
246Error ExecutorSharedMemoryMapperService::release(
247 const std::vector<ExecutorAddr> &Bases) {
248#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
249 Error Err = Error::success();
250
251 for (auto Base : Bases) {
252 std::vector<ExecutorAddr> AllocAddrs;
253 size_t Size;
254
255#if defined(_WIN32)
256 HANDLE SharedMemoryFile;
257#endif
258
259 {
260 std::lock_guard<std::mutex> Lock(Mutex);
261 auto &R = Reservations[Base.toPtr<void *>()];
262 Size = R.Size;
263
264#if defined(_WIN32)
265 SharedMemoryFile = R.SharedMemoryFile;
266#endif
267
268 AllocAddrs.swap(x&: R.Allocations);
269 }
270
271 // deinitialize sub allocations
272 if (Error E = deinitialize(Bases: AllocAddrs))
273 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
274
275#if defined(LLVM_ON_UNIX)
276
277#if defined(__MVS__)
278 (void)Size;
279
280 if (shmdt(Base.toPtr<void *>()) < 0)
281 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
282#else
283 if (munmap(addr: Base.toPtr<void *>(), len: Size) != 0)
284 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
285#endif
286
287#elif defined(_WIN32)
288 (void)Size;
289
290 if (!UnmapViewOfFile(Base.toPtr<void *>()))
291 Err = joinErrors(std::move(Err),
292 errorCodeToError(mapWindowsError(GetLastError())));
293
294 CloseHandle(SharedMemoryFile);
295
296#endif
297
298 std::lock_guard<std::mutex> Lock(Mutex);
299 Reservations.erase(Val: Base.toPtr<void *>());
300 }
301
302 return Err;
303#else
304 return make_error<StringError>(
305 "SharedMemoryMapper is not supported on this platform yet",
306 inconvertibleErrorCode());
307#endif
308}
309
310Error ExecutorSharedMemoryMapperService::shutdown() {
311 if (Reservations.empty())
312 return Error::success();
313
314 std::vector<ExecutorAddr> ReservationAddrs;
315 ReservationAddrs.reserve(n: Reservations.size());
316 for (const auto &R : Reservations)
317 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
318
319 return release(Bases: std::move(ReservationAddrs));
320}
321
322void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
323 StringMap<ExecutorAddr> &M) {
324 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
325 ExecutorAddr::fromPtr(Ptr: this);
326 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
327 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
328 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
329 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
330 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
331 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
332 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
333 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
334}
335
336llvm::orc::shared::CWrapperFunctionResult
337ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
338 size_t ArgSize) {
339 return shared::WrapperFunction<
340 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
341 handle(ArgData, ArgSize,
342 Handler: shared::makeMethodWrapperHandler(
343 Method: &ExecutorSharedMemoryMapperService::reserve))
344 .release();
345}
346
347llvm::orc::shared::CWrapperFunctionResult
348ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
349 size_t ArgSize) {
350 return shared::WrapperFunction<
351 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
352 handle(ArgData, ArgSize,
353 Handler: shared::makeMethodWrapperHandler(
354 Method: &ExecutorSharedMemoryMapperService::initialize))
355 .release();
356}
357
358llvm::orc::shared::CWrapperFunctionResult
359ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
360 size_t ArgSize) {
361 return shared::WrapperFunction<
362 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
363 handle(ArgData, ArgSize,
364 Handler: shared::makeMethodWrapperHandler(
365 Method: &ExecutorSharedMemoryMapperService::deinitialize))
366 .release();
367}
368
369llvm::orc::shared::CWrapperFunctionResult
370ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
371 size_t ArgSize) {
372 return shared::WrapperFunction<
373 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
374 handle(ArgData, ArgSize,
375 Handler: shared::makeMethodWrapperHandler(
376 Method: &ExecutorSharedMemoryMapperService::release))
377 .release();
378}
379
380} // namespace rt_bootstrap
381} // end namespace orc
382} // end namespace llvm
383