1//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10
11#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/Support/MSVCErrorWorkarounds.h"
14#include "llvm/Support/WindowsError.h"
15
16#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
17#include <fcntl.h>
18#include <sys/mman.h>
19#if defined(__MVS__)
20#include "llvm/Support/BLAKE3.h"
21#include <sys/shm.h>
22#endif
23#include <unistd.h>
24#elif defined(_WIN32)
25#include <windows.h>
26#endif
27
28namespace llvm {
29namespace orc {
30
31MemoryMapper::~MemoryMapper() = default;
32
33InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
34 : PageSize(PageSize) {}
35
36Expected<std::unique_ptr<InProcessMemoryMapper>>
37InProcessMemoryMapper::Create() {
38 auto PageSize = sys::Process::getPageSize();
39 if (!PageSize)
40 return PageSize.takeError();
41 return std::make_unique<InProcessMemoryMapper>(args&: *PageSize);
42}
43
44void InProcessMemoryMapper::reserve(size_t NumBytes,
45 OnReservedFunction OnReserved) {
46 std::error_code EC;
47 auto MB = sys::Memory::allocateMappedMemory(
48 NumBytes, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
49
50 if (EC)
51 return OnReserved(errorCodeToError(EC));
52
53 {
54 std::lock_guard<std::mutex> Lock(Mutex);
55 Reservations[MB.base()].Size = MB.allocatedSize();
56 }
57
58 OnReserved(
59 ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: MB.base()), MB.allocatedSize()));
60}
61
62char *InProcessMemoryMapper::prepare(jitlink::LinkGraph &G, ExecutorAddr Addr,
63 size_t ContentSize) {
64 return Addr.toPtr<char *>();
65}
66
67void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
68 OnInitializedFunction OnInitialized) {
69 ExecutorAddr MinAddr(~0ULL);
70 ExecutorAddr MaxAddr(0);
71
72 // FIXME: Release finalize lifetime segments.
73 for (auto &Segment : AI.Segments) {
74 auto Base = AI.MappingBase + Segment.Offset;
75 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
76
77 if (Base < MinAddr)
78 MinAddr = Base;
79
80 if (Base + Size > MaxAddr)
81 MaxAddr = Base + Size;
82
83 std::memset(s: (Base + Segment.ContentSize).toPtr<void *>(), c: 0,
84 n: Segment.ZeroFillSize);
85
86 if (auto EC = sys::Memory::protectMappedMemory(
87 Block: {Base.toPtr<void *>(), Size},
88 Flags: toSysMemoryProtectionFlags(MP: Segment.AG.getMemProt()))) {
89 return OnInitialized(errorCodeToError(EC));
90 }
91 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
92 sys::Memory::InvalidateInstructionCache(Addr: Base.toPtr<void *>(), Len: Size);
93 }
94
95 auto DeinitializeActions = shared::runFinalizeActions(AAs&: AI.Actions);
96 if (!DeinitializeActions)
97 return OnInitialized(DeinitializeActions.takeError());
98
99 {
100 std::lock_guard<std::mutex> Lock(Mutex);
101
102 // This is the maximum range whose permission have been possibly modified
103 auto &Alloc = Allocations[MinAddr];
104 Alloc.Size = MaxAddr - MinAddr;
105 Alloc.DeinitializationActions = std::move(*DeinitializeActions);
106 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(x: MinAddr);
107 }
108
109 OnInitialized(MinAddr);
110}
111
112void InProcessMemoryMapper::deinitialize(
113 ArrayRef<ExecutorAddr> Bases,
114 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
115 Error AllErr = Error::success();
116
117 {
118 std::lock_guard<std::mutex> Lock(Mutex);
119
120 for (auto Base : llvm::reverse(C&: Bases)) {
121
122 if (Error Err = shared::runDeallocActions(
123 DAs: Allocations[Base].DeinitializationActions)) {
124 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
125 }
126
127 // Reset protections to read/write so the area can be reused
128 if (auto EC = sys::Memory::protectMappedMemory(
129 Block: {Base.toPtr<void *>(), Allocations[Base].Size},
130 Flags: sys::Memory::ProtectionFlags::MF_READ |
131 sys::Memory::ProtectionFlags::MF_WRITE)) {
132 AllErr = joinErrors(E1: std::move(AllErr), E2: errorCodeToError(EC));
133 }
134
135 Allocations.erase(Val: Base);
136 }
137 }
138
139 OnDeinitialized(std::move(AllErr));
140}
141
142void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
143 OnReleasedFunction OnReleased) {
144 Error Err = Error::success();
145
146 for (auto Base : Bases) {
147 std::vector<ExecutorAddr> AllocAddrs;
148 size_t Size;
149 {
150 std::lock_guard<std::mutex> Lock(Mutex);
151 auto &R = Reservations[Base.toPtr<void *>()];
152 Size = R.Size;
153 AllocAddrs.swap(x&: R.Allocations);
154 }
155
156 // deinitialize sub allocations
157 std::promise<MSVCPError> P;
158 auto F = P.get_future();
159 deinitialize(Bases: AllocAddrs, OnDeinitialized: [&](Error Err) { P.set_value(std::move(Err)); });
160 if (Error E = F.get()) {
161 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
162 }
163
164 // free the memory
165 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
166
167 auto EC = sys::Memory::releaseMappedMemory(Block&: MB);
168 if (EC) {
169 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
170 }
171
172 std::lock_guard<std::mutex> Lock(Mutex);
173 Reservations.erase(Val: Base.toPtr<void *>());
174 }
175
176 OnReleased(std::move(Err));
177}
178
179InProcessMemoryMapper::~InProcessMemoryMapper() {
180 std::vector<ExecutorAddr> ReservationAddrs;
181 {
182 std::lock_guard<std::mutex> Lock(Mutex);
183
184 ReservationAddrs.reserve(n: Reservations.size());
185 for (const auto &R : Reservations) {
186 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
187 }
188 }
189
190 std::promise<MSVCPError> P;
191 auto F = P.get_future();
192 release(Bases: ReservationAddrs, OnReleased: [&](Error Err) { P.set_value(std::move(Err)); });
193 cantFail(Err: F.get());
194}
195
196// SharedMemoryMapper
197
198SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
199 SymbolAddrs SAs, size_t PageSize)
200 : EPC(EPC), SAs(SAs), PageSize(PageSize) {
201#if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
202 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
203#endif
204}
205
206Expected<std::unique_ptr<SharedMemoryMapper>>
207SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
208#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
209 auto PageSize = sys::Process::getPageSize();
210 if (!PageSize)
211 return PageSize.takeError();
212
213 return std::make_unique<SharedMemoryMapper>(args&: EPC, args&: SAs, args&: *PageSize);
214#else
215 return make_error<StringError>(
216 "SharedMemoryMapper is not supported on this platform yet",
217 inconvertibleErrorCode());
218#endif
219}
220
221void SharedMemoryMapper::reserve(size_t NumBytes,
222 OnReservedFunction OnReserved) {
223#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
224
225 int SharedMemoryId = -1;
226 EPC.callSPSWrapperAsync<
227 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
228 WrapperFnAddr: SAs.Reserve,
229 SendResult: [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId](
230 Error SerializationErr,
231 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
232 if (SerializationErr) {
233 cantFail(Err: Result.takeError());
234 return OnReserved(std::move(SerializationErr));
235 }
236
237 if (!Result)
238 return OnReserved(Result.takeError());
239
240 ExecutorAddr RemoteAddr;
241 std::string SharedMemoryName;
242 std::tie(args&: RemoteAddr, args&: SharedMemoryName) = std::move(*Result);
243
244 void *LocalAddr = nullptr;
245
246#if defined(LLVM_ON_UNIX)
247
248#if defined(__MVS__)
249 ArrayRef<uint8_t> Data(
250 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
251 SharedMemoryName.size());
252 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
253 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
254 SharedMemoryId =
255 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
256 if (SharedMemoryId < 0) {
257 return OnReserved(errorCodeToError(
258 std::error_code(errno, std::generic_category())));
259 }
260 LocalAddr = shmat(SharedMemoryId, nullptr, 0);
261 if (LocalAddr == reinterpret_cast<void *>(-1)) {
262 return OnReserved(errorCodeToError(
263 std::error_code(errno, std::generic_category())));
264 }
265#else
266 int SharedMemoryFile = shm_open(name: SharedMemoryName.c_str(), O_RDWR, mode: 0700);
267 if (SharedMemoryFile < 0) {
268 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
269 }
270
271 // this prevents other processes from accessing it by name
272 shm_unlink(name: SharedMemoryName.c_str());
273
274 LocalAddr = mmap(addr: nullptr, len: NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
275 fd: SharedMemoryFile, offset: 0);
276 if (LocalAddr == MAP_FAILED) {
277 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
278 }
279
280 close(fd: SharedMemoryFile);
281#endif
282
283#elif defined(_WIN32)
284
285 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
286 SharedMemoryName.end());
287 HANDLE SharedMemoryFile = OpenFileMappingW(
288 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
289 if (!SharedMemoryFile)
290 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
291
292 LocalAddr =
293 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
294 if (!LocalAddr) {
295 CloseHandle(SharedMemoryFile);
296 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
297 }
298
299 CloseHandle(SharedMemoryFile);
300
301#endif
302 {
303 std::lock_guard<std::mutex> Lock(Mutex);
304 Reservations.insert(
305 x: {RemoteAddr, {.LocalAddr: LocalAddr, .Size: NumBytes, .SharedMemoryId: SharedMemoryId}});
306 }
307
308 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
309 },
310 Args: SAs.Instance, Args: static_cast<uint64_t>(NumBytes));
311
312#else
313 OnReserved(make_error<StringError>(
314 "SharedMemoryMapper is not supported on this platform yet",
315 inconvertibleErrorCode()));
316#endif
317}
318
319char *SharedMemoryMapper::prepare(jitlink::LinkGraph &G, ExecutorAddr Addr,
320 size_t ContentSize) {
321 auto R = Reservations.upper_bound(x: Addr);
322 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
323 R--;
324
325 ExecutorAddrDiff Offset = Addr - R->first;
326
327 return static_cast<char *>(R->second.LocalAddr) + Offset;
328}
329
330void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
331 OnInitializedFunction OnInitialized) {
332 auto Reservation = Reservations.upper_bound(x: AI.MappingBase);
333 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
334 Reservation--;
335
336 auto AllocationOffset = AI.MappingBase - Reservation->first;
337
338 tpctypes::SharedMemoryFinalizeRequest FR;
339
340 AI.Actions.swap(x&: FR.Actions);
341
342 FR.Segments.reserve(n: AI.Segments.size());
343
344 for (auto Segment : AI.Segments) {
345 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
346 AllocationOffset + Segment.Offset;
347 std::memset(s: Base + Segment.ContentSize, c: 0, n: Segment.ZeroFillSize);
348
349 tpctypes::SharedMemorySegFinalizeRequest SegReq;
350 SegReq.RAG = {Segment.AG.getMemProt(),
351 Segment.AG.getMemLifetime() == MemLifetime::Finalize};
352 SegReq.Addr = AI.MappingBase + Segment.Offset;
353 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
354
355 FR.Segments.push_back(x: SegReq);
356 }
357
358 EPC.callSPSWrapperAsync<
359 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
360 WrapperFnAddr: SAs.Initialize,
361 SendResult: [OnInitialized = std::move(OnInitialized)](
362 Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
363 if (SerializationErr) {
364 cantFail(Err: Result.takeError());
365 return OnInitialized(std::move(SerializationErr));
366 }
367
368 OnInitialized(std::move(Result));
369 },
370 Args: SAs.Instance, Args: Reservation->first, Args: std::move(FR));
371}
372
373void SharedMemoryMapper::deinitialize(
374 ArrayRef<ExecutorAddr> Allocations,
375 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
376 EPC.callSPSWrapperAsync<
377 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
378 WrapperFnAddr: SAs.Deinitialize,
379 SendResult: [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
380 Error Result) mutable {
381 if (SerializationErr) {
382 cantFail(Err: std::move(Result));
383 return OnDeinitialized(std::move(SerializationErr));
384 }
385
386 OnDeinitialized(std::move(Result));
387 },
388 Args: SAs.Instance, Args: Allocations);
389}
390
391void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
392 OnReleasedFunction OnReleased) {
393#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
394 Error Err = Error::success();
395
396 {
397 std::lock_guard<std::mutex> Lock(Mutex);
398
399 for (auto Base : Bases) {
400
401#if defined(LLVM_ON_UNIX)
402
403#if defined(__MVS__)
404 if (shmdt(Reservations[Base].LocalAddr) < 0 ||
405 shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0)
406 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
407#else
408 if (munmap(addr: Reservations[Base].LocalAddr, len: Reservations[Base].Size) != 0)
409 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
410#endif
411
412#elif defined(_WIN32)
413
414 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
415 Err = joinErrors(std::move(Err),
416 errorCodeToError(mapWindowsError(GetLastError())));
417
418#endif
419
420 Reservations.erase(x: Base);
421 }
422 }
423
424 EPC.callSPSWrapperAsync<
425 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
426 WrapperFnAddr: SAs.Release,
427 SendResult: [OnReleased = std::move(OnReleased),
428 Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
429 if (SerializationErr) {
430 cantFail(Err: std::move(Result));
431 return OnReleased(
432 joinErrors(E1: std::move(Err), E2: std::move(SerializationErr)));
433 }
434
435 return OnReleased(joinErrors(E1: std::move(Err), E2: std::move(Result)));
436 },
437 Args: SAs.Instance, Args: Bases);
438#else
439 OnReleased(make_error<StringError>(
440 "SharedMemoryMapper is not supported on this platform yet",
441 inconvertibleErrorCode()));
442#endif
443}
444
445SharedMemoryMapper::~SharedMemoryMapper() {
446 std::lock_guard<std::mutex> Lock(Mutex);
447 for (const auto &R : Reservations) {
448
449#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
450
451#if defined(__MVS__)
452 shmdt(R.second.LocalAddr);
453#else
454 munmap(addr: R.second.LocalAddr, len: R.second.Size);
455#endif
456
457#elif defined(_WIN32)
458
459 UnmapViewOfFile(R.second.LocalAddr);
460
461#else
462
463 (void)R;
464
465#endif
466 }
467}
468
469} // namespace orc
470
471} // namespace llvm
472