1//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
11#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12#include "llvm/Support/Process.h"
13#include "llvm/Support/WindowsError.h"
14#include <sstream>
15
16#if defined(LLVM_ON_UNIX)
17#include <errno.h>
18#include <fcntl.h>
19#include <sys/mman.h>
20#if defined(__MVS__)
21#include "llvm/Support/BLAKE3.h"
22#include <sys/shm.h>
23#endif
24#include <unistd.h>
25#endif
26
27namespace llvm {
28namespace orc {
29namespace rt_bootstrap {
30
31#if defined(_WIN32)
32static DWORD getWindowsProtectionFlags(MemProt MP) {
33 if (MP == MemProt::Read)
34 return PAGE_READONLY;
35 if (MP == MemProt::Write ||
36 MP == (MemProt::Write | MemProt::Read)) {
37 // Note: PAGE_WRITE is not supported by VirtualProtect
38 return PAGE_READWRITE;
39 }
40 if (MP == (MemProt::Read | MemProt::Exec))
41 return PAGE_EXECUTE_READ;
42 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
43 return PAGE_EXECUTE_READWRITE;
44 if (MP == MemProt::Exec)
45 return PAGE_EXECUTE;
46
47 return PAGE_NOACCESS;
48}
49#endif
50
51Expected<std::pair<ExecutorAddr, std::string>>
52ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
53#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
54
55#if defined(LLVM_ON_UNIX)
56
57 std::string SharedMemoryName;
58 {
59 std::stringstream SharedMemoryNameStream;
60 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
61 << (++SharedMemoryCount);
62 SharedMemoryName = SharedMemoryNameStream.str();
63 }
64
65#if defined(__MVS__)
66 ArrayRef<uint8_t> Data(
67 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
68 SharedMemoryName.size());
69 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
70 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
71 int SharedMemoryId =
72 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
73 if (SharedMemoryId < 0)
74 return errorCodeToError(errnoAsErrorCode());
75
76 void *Addr = shmat(SharedMemoryId, nullptr, 0);
77 if (Addr == reinterpret_cast<void *>(-1))
78 return errorCodeToError(errnoAsErrorCode());
79#else
80 int SharedMemoryFile =
81 shm_open(name: SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, mode: 0700);
82 if (SharedMemoryFile < 0)
83 return errorCodeToError(EC: errnoAsErrorCode());
84
85 // by default size is 0
86 if (ftruncate(fd: SharedMemoryFile, length: Size) < 0)
87 return errorCodeToError(EC: errnoAsErrorCode());
88
89 void *Addr = mmap(addr: nullptr, len: Size, PROT_NONE, MAP_SHARED, fd: SharedMemoryFile, offset: 0);
90 if (Addr == MAP_FAILED)
91 return errorCodeToError(EC: errnoAsErrorCode());
92
93 close(fd: SharedMemoryFile);
94#endif
95
96#elif defined(_WIN32)
97
98 std::string SharedMemoryName;
99 {
100 std::stringstream SharedMemoryNameStream;
101 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
102 << (++SharedMemoryCount);
103 SharedMemoryName = SharedMemoryNameStream.str();
104 }
105
106 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
107 SharedMemoryName.end());
108 HANDLE SharedMemoryFile = CreateFileMappingW(
109 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
110 Size & 0xffffffff, WideSharedMemoryName.c_str());
111 if (!SharedMemoryFile)
112 return errorCodeToError(mapWindowsError(GetLastError()));
113
114 void *Addr = MapViewOfFile(SharedMemoryFile,
115 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
116 if (!Addr) {
117 CloseHandle(SharedMemoryFile);
118 return errorCodeToError(mapWindowsError(GetLastError()));
119 }
120
121#endif
122
123 {
124 std::lock_guard<std::mutex> Lock(Mutex);
125 Reservations[Addr].Size = Size;
126#if defined(_WIN32)
127 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
128#endif
129 }
130
131 return std::make_pair(x: ExecutorAddr::fromPtr(Ptr: Addr),
132 y: std::move(SharedMemoryName));
133#else
134 return make_error<StringError>(
135 "SharedMemoryMapper is not supported on this platform yet",
136 inconvertibleErrorCode());
137#endif
138}
139
140Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
141 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
142#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
143
144 ExecutorAddr MinAddr(~0ULL);
145
146 // Contents are already in place
147 for (auto &Segment : FR.Segments) {
148 if (Segment.Addr < MinAddr)
149 MinAddr = Segment.Addr;
150
151#if defined(LLVM_ON_UNIX)
152
153#if defined(__MVS__)
154 // TODO Is it possible to change the protection level?
155#else
156 int NativeProt = 0;
157 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
158 NativeProt |= PROT_READ;
159 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
160 NativeProt |= PROT_WRITE;
161 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
162 NativeProt |= PROT_EXEC;
163
164 if (mprotect(addr: Segment.Addr.toPtr<void *>(), len: Segment.Size, prot: NativeProt))
165 return errorCodeToError(EC: errnoAsErrorCode());
166#endif
167
168#elif defined(_WIN32)
169
170 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
171
172 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
173 &NativeProt))
174 return errorCodeToError(mapWindowsError(GetLastError()));
175
176#endif
177
178 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
179 sys::Memory::InvalidateInstructionCache(Addr: Segment.Addr.toPtr<void *>(),
180 Len: Segment.Size);
181 }
182
183 // Run finalization actions and get deinitlization action list.
184 auto DeinitializeActions = shared::runFinalizeActions(AAs&: FR.Actions);
185 if (!DeinitializeActions) {
186 return DeinitializeActions.takeError();
187 }
188
189 {
190 std::lock_guard<std::mutex> Lock(Mutex);
191 Allocations[MinAddr].DeinitializationActions =
192 std::move(*DeinitializeActions);
193 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(x: MinAddr);
194 }
195
196 return MinAddr;
197
198#else
199 return make_error<StringError>(
200 "SharedMemoryMapper is not supported on this platform yet",
201 inconvertibleErrorCode());
202#endif
203}
204
205Error ExecutorSharedMemoryMapperService::deinitialize(
206 const std::vector<ExecutorAddr> &Bases) {
207 Error AllErr = Error::success();
208
209 {
210 std::lock_guard<std::mutex> Lock(Mutex);
211
212 for (auto Base : llvm::reverse(C: Bases)) {
213 if (Error Err = shared::runDeallocActions(
214 DAs: Allocations[Base].DeinitializationActions)) {
215 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
216 }
217
218 // Remove the allocation from the allocation list of its reservation
219 for (auto &Reservation : Reservations) {
220 auto AllocationIt = llvm::find(Range&: Reservation.second.Allocations, Val: Base);
221 if (AllocationIt != Reservation.second.Allocations.end()) {
222 Reservation.second.Allocations.erase(position: AllocationIt);
223 break;
224 }
225 }
226
227 Allocations.erase(Val: Base);
228 }
229 }
230
231 return AllErr;
232}
233
234Error ExecutorSharedMemoryMapperService::release(
235 const std::vector<ExecutorAddr> &Bases) {
236#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
237 Error Err = Error::success();
238
239 for (auto Base : Bases) {
240 std::vector<ExecutorAddr> AllocAddrs;
241 size_t Size;
242
243#if defined(_WIN32)
244 HANDLE SharedMemoryFile;
245#endif
246
247 {
248 std::lock_guard<std::mutex> Lock(Mutex);
249 auto &R = Reservations[Base.toPtr<void *>()];
250 Size = R.Size;
251
252#if defined(_WIN32)
253 SharedMemoryFile = R.SharedMemoryFile;
254#endif
255
256 AllocAddrs.swap(x&: R.Allocations);
257 }
258
259 // deinitialize sub allocations
260 if (Error E = deinitialize(Bases: AllocAddrs))
261 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
262
263#if defined(LLVM_ON_UNIX)
264
265#if defined(__MVS__)
266 (void)Size;
267
268 if (shmdt(Base.toPtr<void *>()) < 0)
269 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
270#else
271 if (munmap(addr: Base.toPtr<void *>(), len: Size) != 0)
272 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
273#endif
274
275#elif defined(_WIN32)
276 (void)Size;
277
278 if (!UnmapViewOfFile(Base.toPtr<void *>()))
279 Err = joinErrors(std::move(Err),
280 errorCodeToError(mapWindowsError(GetLastError())));
281
282 CloseHandle(SharedMemoryFile);
283
284#endif
285
286 std::lock_guard<std::mutex> Lock(Mutex);
287 Reservations.erase(Val: Base.toPtr<void *>());
288 }
289
290 return Err;
291#else
292 return make_error<StringError>(
293 "SharedMemoryMapper is not supported on this platform yet",
294 inconvertibleErrorCode());
295#endif
296}
297
298Error ExecutorSharedMemoryMapperService::shutdown() {
299 if (Reservations.empty())
300 return Error::success();
301
302 std::vector<ExecutorAddr> ReservationAddrs;
303 ReservationAddrs.reserve(n: Reservations.size());
304 for (const auto &R : Reservations)
305 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
306
307 return release(Bases: std::move(ReservationAddrs));
308}
309
310void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
311 StringMap<ExecutorAddr> &M) {
312 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
313 ExecutorAddr::fromPtr(Ptr: this);
314 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
315 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
316 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
317 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
318 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
319 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
320 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
321 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
322}
323
324llvm::orc::shared::CWrapperFunctionBuffer
325ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
326 size_t ArgSize) {
327 return shared::WrapperFunction<
328 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
329 handle(ArgData, ArgSize,
330 Handler: shared::makeMethodWrapperHandler(
331 Method: &ExecutorSharedMemoryMapperService::reserve))
332 .release();
333}
334
335llvm::orc::shared::CWrapperFunctionBuffer
336ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
337 size_t ArgSize) {
338 return shared::WrapperFunction<
339 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
340 handle(ArgData, ArgSize,
341 Handler: shared::makeMethodWrapperHandler(
342 Method: &ExecutorSharedMemoryMapperService::initialize))
343 .release();
344}
345
346llvm::orc::shared::CWrapperFunctionBuffer
347ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
348 size_t ArgSize) {
349 return shared::WrapperFunction<
350 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
351 handle(ArgData, ArgSize,
352 Handler: shared::makeMethodWrapperHandler(
353 Method: &ExecutorSharedMemoryMapperService::deinitialize))
354 .release();
355}
356
357llvm::orc::shared::CWrapperFunctionBuffer
358ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
359 size_t ArgSize) {
360 return shared::WrapperFunction<
361 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
362 handle(ArgData, ArgSize,
363 Handler: shared::makeMethodWrapperHandler(
364 Method: &ExecutorSharedMemoryMapperService::release))
365 .release();
366}
367
368} // namespace rt_bootstrap
369} // end namespace orc
370} // end namespace llvm
371