1//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
10#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
11#include "llvm/ExecutionEngine/Orc/Shared/Mangler.h"
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/ExecutionEngine/Orc/Shared/SPSCI/SharedMemoryMapperSPSCI.h"
14#include "llvm/Support/Process.h"
15#include "llvm/Support/WindowsError.h"
16#include "llvm/TargetParser/Host.h"
17#include "llvm/TargetParser/Triple.h"
18#include <sstream>
19
20#if defined(LLVM_ON_UNIX)
21#include <errno.h>
22#include <fcntl.h>
23#include <sys/mman.h>
24#if defined(__MVS__)
25#include "llvm/Support/BLAKE3.h"
26#include <sys/shm.h>
27#endif
28#include <unistd.h>
29#endif
30
31namespace llvm {
32namespace orc {
33namespace rt_bootstrap {
34
35#if defined(_WIN32)
36static DWORD getWindowsProtectionFlags(MemProt MP) {
37 if (MP == MemProt::Read)
38 return PAGE_READONLY;
39 if (MP == MemProt::Write ||
40 MP == (MemProt::Write | MemProt::Read)) {
41 // Note: PAGE_WRITE is not supported by VirtualProtect
42 return PAGE_READWRITE;
43 }
44 if (MP == (MemProt::Read | MemProt::Exec))
45 return PAGE_EXECUTE_READ;
46 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
47 return PAGE_EXECUTE_READWRITE;
48 if (MP == MemProt::Exec)
49 return PAGE_EXECUTE;
50
51 return PAGE_NOACCESS;
52}
53#endif
54
55Expected<std::pair<ExecutorAddr, std::string>>
56ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
57#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
58
59#if defined(LLVM_ON_UNIX)
60
61 std::string SharedMemoryName;
62 {
63 std::stringstream SharedMemoryNameStream;
64 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
65 << (++SharedMemoryCount);
66 SharedMemoryName = SharedMemoryNameStream.str();
67 }
68
69#if defined(__MVS__)
70 ArrayRef<uint8_t> Data(
71 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
72 SharedMemoryName.size());
73 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
74 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
75 int SharedMemoryId =
76 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
77 if (SharedMemoryId < 0)
78 return errorCodeToError(errnoAsErrorCode());
79
80 void *Addr = shmat(SharedMemoryId, nullptr, 0);
81 if (Addr == reinterpret_cast<void *>(-1))
82 return errorCodeToError(errnoAsErrorCode());
83#else
84 int SharedMemoryFile =
85 shm_open(name: SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, mode: 0700);
86 if (SharedMemoryFile < 0)
87 return errorCodeToError(EC: errnoAsErrorCode());
88
89 // by default size is 0
90 if (ftruncate(fd: SharedMemoryFile, length: Size) < 0)
91 return errorCodeToError(EC: errnoAsErrorCode());
92
93 void *Addr = mmap(addr: nullptr, len: Size, PROT_NONE, MAP_SHARED, fd: SharedMemoryFile, offset: 0);
94 if (Addr == MAP_FAILED)
95 return errorCodeToError(EC: errnoAsErrorCode());
96
97 close(fd: SharedMemoryFile);
98#endif
99
100#elif defined(_WIN32)
101
102 std::string SharedMemoryName;
103 {
104 std::stringstream SharedMemoryNameStream;
105 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
106 << (++SharedMemoryCount);
107 SharedMemoryName = SharedMemoryNameStream.str();
108 }
109
110 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
111 SharedMemoryName.end());
112 HANDLE SharedMemoryFile = CreateFileMappingW(
113 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
114 Size & 0xffffffff, WideSharedMemoryName.c_str());
115 if (!SharedMemoryFile)
116 return errorCodeToError(mapWindowsError(GetLastError()));
117
118 void *Addr = MapViewOfFile(SharedMemoryFile,
119 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
120 if (!Addr) {
121 CloseHandle(SharedMemoryFile);
122 return errorCodeToError(mapWindowsError(GetLastError()));
123 }
124
125#endif
126
127 {
128 std::lock_guard<std::mutex> Lock(Mutex);
129 Reservations[Addr].Size = Size;
130#if defined(_WIN32)
131 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
132#endif
133 }
134
135 return std::make_pair(x: ExecutorAddr::fromPtr(Ptr: Addr),
136 y: std::move(SharedMemoryName));
137#else
138 return make_error<StringError>(
139 "SharedMemoryMapper is not supported on this platform yet",
140 inconvertibleErrorCode());
141#endif
142}
143
144Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
145 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
146#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
147
148 ExecutorAddr MinAddr(~0ULL);
149
150 // Contents are already in place
151 for (auto &Segment : FR.Segments) {
152 if (Segment.Addr < MinAddr)
153 MinAddr = Segment.Addr;
154
155#if defined(LLVM_ON_UNIX)
156
157#if defined(__MVS__)
158 // TODO Is it possible to change the protection level?
159#else
160 int NativeProt = 0;
161 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
162 NativeProt |= PROT_READ;
163 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
164 NativeProt |= PROT_WRITE;
165 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
166 NativeProt |= PROT_EXEC;
167
168 if (mprotect(addr: Segment.Addr.toPtr<void *>(), len: Segment.Size, prot: NativeProt))
169 return errorCodeToError(EC: errnoAsErrorCode());
170#endif
171
172#elif defined(_WIN32)
173
174 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
175
176 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
177 &NativeProt))
178 return errorCodeToError(mapWindowsError(GetLastError()));
179
180#endif
181
182 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
183 sys::Memory::InvalidateInstructionCache(Addr: Segment.Addr.toPtr<void *>(),
184 Len: Segment.Size);
185 }
186
187 // Run finalization actions and get deinitlization action list.
188 auto DeinitializeActions = shared::runFinalizeActions(AAs&: FR.Actions);
189 if (!DeinitializeActions) {
190 return DeinitializeActions.takeError();
191 }
192
193 {
194 std::lock_guard<std::mutex> Lock(Mutex);
195 Allocations[MinAddr].DeinitializationActions =
196 std::move(*DeinitializeActions);
197 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(x: MinAddr);
198 }
199
200 return MinAddr;
201
202#else
203 return make_error<StringError>(
204 "SharedMemoryMapper is not supported on this platform yet",
205 inconvertibleErrorCode());
206#endif
207}
208
209Error ExecutorSharedMemoryMapperService::deinitialize(
210 const std::vector<ExecutorAddr> &Bases) {
211 Error AllErr = Error::success();
212
213 {
214 std::lock_guard<std::mutex> Lock(Mutex);
215
216 for (auto Base : llvm::reverse(C: Bases)) {
217 if (Error Err = shared::runDeallocActions(
218 DAs: Allocations[Base].DeinitializationActions)) {
219 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
220 }
221
222 // Remove the allocation from the allocation list of its reservation
223 for (auto &Reservation : Reservations) {
224 auto AllocationIt = llvm::find(Range&: Reservation.second.Allocations, Val: Base);
225 if (AllocationIt != Reservation.second.Allocations.end()) {
226 Reservation.second.Allocations.erase(position: AllocationIt);
227 break;
228 }
229 }
230
231 Allocations.erase(Val: Base);
232 }
233 }
234
235 return AllErr;
236}
237
238Error ExecutorSharedMemoryMapperService::release(
239 const std::vector<ExecutorAddr> &Bases) {
240#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
241 Error Err = Error::success();
242
243 for (auto Base : Bases) {
244 std::vector<ExecutorAddr> AllocAddrs;
245 size_t Size;
246
247#if defined(_WIN32)
248 HANDLE SharedMemoryFile;
249#endif
250
251 {
252 std::lock_guard<std::mutex> Lock(Mutex);
253 auto &R = Reservations[Base.toPtr<void *>()];
254 Size = R.Size;
255
256#if defined(_WIN32)
257 SharedMemoryFile = R.SharedMemoryFile;
258#endif
259
260 AllocAddrs.swap(x&: R.Allocations);
261 }
262
263 // deinitialize sub allocations
264 if (Error E = deinitialize(Bases: AllocAddrs))
265 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
266
267#if defined(LLVM_ON_UNIX)
268
269#if defined(__MVS__)
270 (void)Size;
271
272 if (shmdt(Base.toPtr<void *>()) < 0)
273 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
274#else
275 if (munmap(addr: Base.toPtr<void *>(), len: Size) != 0)
276 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
277#endif
278
279#elif defined(_WIN32)
280 (void)Size;
281
282 if (!UnmapViewOfFile(Base.toPtr<void *>()))
283 Err = joinErrors(std::move(Err),
284 errorCodeToError(mapWindowsError(GetLastError())));
285
286 CloseHandle(SharedMemoryFile);
287
288#endif
289
290 std::lock_guard<std::mutex> Lock(Mutex);
291 Reservations.erase(Val: Base.toPtr<void *>());
292 }
293
294 return Err;
295#else
296 return make_error<StringError>(
297 "SharedMemoryMapper is not supported on this platform yet",
298 inconvertibleErrorCode());
299#endif
300}
301
302Error ExecutorSharedMemoryMapperService::shutdown() {
303 if (Reservations.empty())
304 return Error::success();
305
306 std::vector<ExecutorAddr> ReservationAddrs;
307 ReservationAddrs.reserve(n: Reservations.size());
308 for (const auto &R : Reservations)
309 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
310
311 return release(Bases: std::move(ReservationAddrs));
312}
313
314void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
315 StringMap<ExecutorAddr> &M) {
316 Mangler Mangle{Triple(sys::getProcessTriple())};
317 M[Mangle.mangledCopy(Name: rt::sps_ci::SharedMemoryMapperInstanceName)] =
318 ExecutorAddr::fromPtr(Ptr: this);
319 M[Mangle.mangledCopy(Name: rt::sps_ci::SharedMemoryMapperReserve::Name)] =
320 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
321 M[Mangle.mangledCopy(Name: rt::sps_ci::SharedMemoryMapperInitialize::Name)] =
322 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
323 M[Mangle.mangledCopy(Name: rt::sps_ci::SharedMemoryMapperDeinitialize::Name)] =
324 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
325 M[Mangle.mangledCopy(Name: rt::sps_ci::SharedMemoryMapperRelease::Name)] =
326 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
327}
328
329llvm::orc::shared::CWrapperFunctionBuffer
330ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
331 size_t ArgSize) {
332 return shared::
333 WrapperFunction<rt::sps_ci::SharedMemoryMapperReserve::SPSSig>::handle(
334 ArgData, ArgSize,
335 Handler: shared::makeMethodWrapperHandler(
336 Method: &ExecutorSharedMemoryMapperService::reserve))
337 .release();
338}
339
340llvm::orc::shared::CWrapperFunctionBuffer
341ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
342 size_t ArgSize) {
343 return shared::
344 WrapperFunction<rt::sps_ci::SharedMemoryMapperInitialize::SPSSig>::handle(
345 ArgData, ArgSize,
346 Handler: shared::makeMethodWrapperHandler(
347 Method: &ExecutorSharedMemoryMapperService::initialize))
348 .release();
349}
350
351llvm::orc::shared::CWrapperFunctionBuffer
352ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
353 size_t ArgSize) {
354 return shared::WrapperFunction<
355 rt::sps_ci::SharedMemoryMapperDeinitialize::SPSSig>::
356 handle(ArgData, ArgSize,
357 Handler: shared::makeMethodWrapperHandler(
358 Method: &ExecutorSharedMemoryMapperService::deinitialize))
359 .release();
360}
361
362llvm::orc::shared::CWrapperFunctionBuffer
363ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
364 size_t ArgSize) {
365 return shared::
366 WrapperFunction<rt::sps_ci::SharedMemoryMapperRelease::SPSSig>::handle(
367 ArgData, ArgSize,
368 Handler: shared::makeMethodWrapperHandler(
369 Method: &ExecutorSharedMemoryMapperService::release))
370 .release();
371}
372
373} // namespace rt_bootstrap
374} // end namespace orc
375} // end namespace llvm
376