1//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10
11#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12#include "llvm/Support/Process.h"
13#include "llvm/Support/WindowsError.h"
14
15#include <sstream>
16
17#if defined(LLVM_ON_UNIX)
18#include <errno.h>
19#include <fcntl.h>
20#include <sys/mman.h>
21#if defined(__MVS__)
22#include "llvm/Support/BLAKE3.h"
23#include <sys/shm.h>
24#endif
25#include <unistd.h>
26#endif
27
28namespace llvm {
29namespace orc {
30namespace rt_bootstrap {
31
32#if defined(_WIN32)
33static DWORD getWindowsProtectionFlags(MemProt MP) {
34 if (MP == MemProt::Read)
35 return PAGE_READONLY;
36 if (MP == MemProt::Write ||
37 MP == (MemProt::Write | MemProt::Read)) {
38 // Note: PAGE_WRITE is not supported by VirtualProtect
39 return PAGE_READWRITE;
40 }
41 if (MP == (MemProt::Read | MemProt::Exec))
42 return PAGE_EXECUTE_READ;
43 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
44 return PAGE_EXECUTE_READWRITE;
45 if (MP == MemProt::Exec)
46 return PAGE_EXECUTE;
47
48 return PAGE_NOACCESS;
49}
50#endif
51
52Expected<std::pair<ExecutorAddr, std::string>>
53ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
54#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
55
56#if defined(LLVM_ON_UNIX)
57
58 std::string SharedMemoryName;
59 {
60 std::stringstream SharedMemoryNameStream;
61 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
62 << (++SharedMemoryCount);
63 SharedMemoryName = SharedMemoryNameStream.str();
64 }
65
66#if defined(__MVS__)
67 ArrayRef<uint8_t> Data(
68 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
69 SharedMemoryName.size());
70 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
71 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
72 int SharedMemoryId =
73 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
74 if (SharedMemoryId < 0)
75 return errorCodeToError(errnoAsErrorCode());
76
77 void *Addr = shmat(SharedMemoryId, nullptr, 0);
78 if (Addr == reinterpret_cast<void *>(-1))
79 return errorCodeToError(errnoAsErrorCode());
80#else
81 int SharedMemoryFile =
82 shm_open(name: SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, mode: 0700);
83 if (SharedMemoryFile < 0)
84 return errorCodeToError(EC: errnoAsErrorCode());
85
86 // by default size is 0
87 if (ftruncate(fd: SharedMemoryFile, length: Size) < 0)
88 return errorCodeToError(EC: errnoAsErrorCode());
89
90 void *Addr = mmap(addr: nullptr, len: Size, PROT_NONE, MAP_SHARED, fd: SharedMemoryFile, offset: 0);
91 if (Addr == MAP_FAILED)
92 return errorCodeToError(EC: errnoAsErrorCode());
93
94 close(fd: SharedMemoryFile);
95#endif
96
97#elif defined(_WIN32)
98
99 std::string SharedMemoryName;
100 {
101 std::stringstream SharedMemoryNameStream;
102 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
103 << (++SharedMemoryCount);
104 SharedMemoryName = SharedMemoryNameStream.str();
105 }
106
107 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
108 SharedMemoryName.end());
109 HANDLE SharedMemoryFile = CreateFileMappingW(
110 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
111 Size & 0xffffffff, WideSharedMemoryName.c_str());
112 if (!SharedMemoryFile)
113 return errorCodeToError(mapWindowsError(GetLastError()));
114
115 void *Addr = MapViewOfFile(SharedMemoryFile,
116 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
117 if (!Addr) {
118 CloseHandle(SharedMemoryFile);
119 return errorCodeToError(mapWindowsError(GetLastError()));
120 }
121
122#endif
123
124 {
125 std::lock_guard<std::mutex> Lock(Mutex);
126 Reservations[Addr].Size = Size;
127#if defined(_WIN32)
128 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
129#endif
130 }
131
132 return std::make_pair(x: ExecutorAddr::fromPtr(Ptr: Addr),
133 y: std::move(SharedMemoryName));
134#else
135 return make_error<StringError>(
136 "SharedMemoryMapper is not supported on this platform yet",
137 inconvertibleErrorCode());
138#endif
139}
140
141Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
142 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
143#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
144
145 ExecutorAddr MinAddr(~0ULL);
146
147 // Contents are already in place
148 for (auto &Segment : FR.Segments) {
149 if (Segment.Addr < MinAddr)
150 MinAddr = Segment.Addr;
151
152#if defined(LLVM_ON_UNIX)
153
154#if defined(__MVS__)
155 // TODO Is it possible to change the protection level?
156#else
157 int NativeProt = 0;
158 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
159 NativeProt |= PROT_READ;
160 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
161 NativeProt |= PROT_WRITE;
162 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
163 NativeProt |= PROT_EXEC;
164
165 if (mprotect(addr: Segment.Addr.toPtr<void *>(), len: Segment.Size, prot: NativeProt))
166 return errorCodeToError(EC: errnoAsErrorCode());
167#endif
168
169#elif defined(_WIN32)
170
171 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
172
173 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
174 &NativeProt))
175 return errorCodeToError(mapWindowsError(GetLastError()));
176
177#endif
178
179 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
180 sys::Memory::InvalidateInstructionCache(Addr: Segment.Addr.toPtr<void *>(),
181 Len: Segment.Size);
182 }
183
184 // Run finalization actions and get deinitlization action list.
185 auto DeinitializeActions = shared::runFinalizeActions(AAs&: FR.Actions);
186 if (!DeinitializeActions) {
187 return DeinitializeActions.takeError();
188 }
189
190 {
191 std::lock_guard<std::mutex> Lock(Mutex);
192 Allocations[MinAddr].DeinitializationActions =
193 std::move(*DeinitializeActions);
194 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(x: MinAddr);
195 }
196
197 return MinAddr;
198
199#else
200 return make_error<StringError>(
201 "SharedMemoryMapper is not supported on this platform yet",
202 inconvertibleErrorCode());
203#endif
204}
205
206Error ExecutorSharedMemoryMapperService::deinitialize(
207 const std::vector<ExecutorAddr> &Bases) {
208 Error AllErr = Error::success();
209
210 {
211 std::lock_guard<std::mutex> Lock(Mutex);
212
213 for (auto Base : llvm::reverse(C: Bases)) {
214 if (Error Err = shared::runDeallocActions(
215 DAs: Allocations[Base].DeinitializationActions)) {
216 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
217 }
218
219 // Remove the allocation from the allocation list of its reservation
220 for (auto &Reservation : Reservations) {
221 auto AllocationIt = llvm::find(Range&: Reservation.second.Allocations, Val: Base);
222 if (AllocationIt != Reservation.second.Allocations.end()) {
223 Reservation.second.Allocations.erase(position: AllocationIt);
224 break;
225 }
226 }
227
228 Allocations.erase(Val: Base);
229 }
230 }
231
232 return AllErr;
233}
234
235Error ExecutorSharedMemoryMapperService::release(
236 const std::vector<ExecutorAddr> &Bases) {
237#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
238 Error Err = Error::success();
239
240 for (auto Base : Bases) {
241 std::vector<ExecutorAddr> AllocAddrs;
242 size_t Size;
243
244#if defined(_WIN32)
245 HANDLE SharedMemoryFile;
246#endif
247
248 {
249 std::lock_guard<std::mutex> Lock(Mutex);
250 auto &R = Reservations[Base.toPtr<void *>()];
251 Size = R.Size;
252
253#if defined(_WIN32)
254 SharedMemoryFile = R.SharedMemoryFile;
255#endif
256
257 AllocAddrs.swap(x&: R.Allocations);
258 }
259
260 // deinitialize sub allocations
261 if (Error E = deinitialize(Bases: AllocAddrs))
262 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
263
264#if defined(LLVM_ON_UNIX)
265
266#if defined(__MVS__)
267 (void)Size;
268
269 if (shmdt(Base.toPtr<void *>()) < 0)
270 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
271#else
272 if (munmap(addr: Base.toPtr<void *>(), len: Size) != 0)
273 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
274#endif
275
276#elif defined(_WIN32)
277 (void)Size;
278
279 if (!UnmapViewOfFile(Base.toPtr<void *>()))
280 Err = joinErrors(std::move(Err),
281 errorCodeToError(mapWindowsError(GetLastError())));
282
283 CloseHandle(SharedMemoryFile);
284
285#endif
286
287 std::lock_guard<std::mutex> Lock(Mutex);
288 Reservations.erase(Val: Base.toPtr<void *>());
289 }
290
291 return Err;
292#else
293 return make_error<StringError>(
294 "SharedMemoryMapper is not supported on this platform yet",
295 inconvertibleErrorCode());
296#endif
297}
298
299Error ExecutorSharedMemoryMapperService::shutdown() {
300 if (Reservations.empty())
301 return Error::success();
302
303 std::vector<ExecutorAddr> ReservationAddrs;
304 ReservationAddrs.reserve(n: Reservations.size());
305 for (const auto &R : Reservations)
306 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
307
308 return release(Bases: std::move(ReservationAddrs));
309}
310
311void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
312 StringMap<ExecutorAddr> &M) {
313 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
314 ExecutorAddr::fromPtr(Ptr: this);
315 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
316 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
317 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
318 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
319 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
320 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
321 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
322 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
323}
324
325llvm::orc::shared::CWrapperFunctionResult
326ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
327 size_t ArgSize) {
328 return shared::WrapperFunction<
329 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
330 handle(ArgData, ArgSize,
331 Handler: shared::makeMethodWrapperHandler(
332 Method: &ExecutorSharedMemoryMapperService::reserve))
333 .release();
334}
335
336llvm::orc::shared::CWrapperFunctionResult
337ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
338 size_t ArgSize) {
339 return shared::WrapperFunction<
340 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
341 handle(ArgData, ArgSize,
342 Handler: shared::makeMethodWrapperHandler(
343 Method: &ExecutorSharedMemoryMapperService::initialize))
344 .release();
345}
346
347llvm::orc::shared::CWrapperFunctionResult
348ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
349 size_t ArgSize) {
350 return shared::WrapperFunction<
351 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
352 handle(ArgData, ArgSize,
353 Handler: shared::makeMethodWrapperHandler(
354 Method: &ExecutorSharedMemoryMapperService::deinitialize))
355 .release();
356}
357
358llvm::orc::shared::CWrapperFunctionResult
359ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
360 size_t ArgSize) {
361 return shared::WrapperFunction<
362 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
363 handle(ArgData, ArgSize,
364 Handler: shared::makeMethodWrapperHandler(
365 Method: &ExecutorSharedMemoryMapperService::release))
366 .release();
367}
368
369} // namespace rt_bootstrap
370} // end namespace orc
371} // end namespace llvm
372