1//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10
11#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
12#include "llvm/Support/MSVCErrorWorkarounds.h"
13#include "llvm/Support/WindowsError.h"
14
15#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
16#include <fcntl.h>
17#include <sys/mman.h>
18#if defined(__MVS__)
19#include "llvm/Support/BLAKE3.h"
20#include <sys/shm.h>
21#endif
22#include <unistd.h>
23#elif defined(_WIN32)
24#include <windows.h>
25#endif
26
27namespace llvm {
28namespace orc {
29
30MemoryMapper::~MemoryMapper() = default;
31
32InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
33 : PageSize(PageSize) {}
34
35Expected<std::unique_ptr<InProcessMemoryMapper>>
36InProcessMemoryMapper::Create() {
37 auto PageSize = sys::Process::getPageSize();
38 if (!PageSize)
39 return PageSize.takeError();
40 return std::make_unique<InProcessMemoryMapper>(args&: *PageSize);
41}
42
43void InProcessMemoryMapper::reserve(size_t NumBytes,
44 OnReservedFunction OnReserved) {
45 std::error_code EC;
46 auto MB = sys::Memory::allocateMappedMemory(
47 NumBytes, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
48
49 if (EC)
50 return OnReserved(errorCodeToError(EC));
51
52 {
53 std::lock_guard<std::mutex> Lock(Mutex);
54 Reservations[MB.base()].Size = MB.allocatedSize();
55 }
56
57 OnReserved(
58 ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: MB.base()), MB.allocatedSize()));
59}
60
61char *InProcessMemoryMapper::prepare(jitlink::LinkGraph &G, ExecutorAddr Addr,
62 size_t ContentSize) {
63 return Addr.toPtr<char *>();
64}
65
66void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
67 OnInitializedFunction OnInitialized) {
68 ExecutorAddr MinAddr(~0ULL);
69 ExecutorAddr MaxAddr(0);
70
71 // FIXME: Release finalize lifetime segments.
72 for (auto &Segment : AI.Segments) {
73 auto Base = AI.MappingBase + Segment.Offset;
74 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
75
76 if (Base < MinAddr)
77 MinAddr = Base;
78
79 if (Base + Size > MaxAddr)
80 MaxAddr = Base + Size;
81
82 std::memset(s: (Base + Segment.ContentSize).toPtr<void *>(), c: 0,
83 n: Segment.ZeroFillSize);
84
85 if (auto EC = sys::Memory::protectMappedMemory(
86 Block: {Base.toPtr<void *>(), Size},
87 Flags: toSysMemoryProtectionFlags(MP: Segment.AG.getMemProt()))) {
88 return OnInitialized(errorCodeToError(EC));
89 }
90 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
91 sys::Memory::InvalidateInstructionCache(Addr: Base.toPtr<void *>(), Len: Size);
92 }
93
94 auto DeinitializeActions = shared::runFinalizeActions(AAs&: AI.Actions);
95 if (!DeinitializeActions)
96 return OnInitialized(DeinitializeActions.takeError());
97
98 {
99 std::lock_guard<std::mutex> Lock(Mutex);
100
101 // This is the maximum range whose permission have been possibly modified
102 auto &Alloc = Allocations[MinAddr];
103 Alloc.Size = MaxAddr - MinAddr;
104 Alloc.DeinitializationActions = std::move(*DeinitializeActions);
105 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(x: MinAddr);
106 }
107
108 OnInitialized(MinAddr);
109}
110
111void InProcessMemoryMapper::deinitialize(
112 ArrayRef<ExecutorAddr> Bases,
113 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
114 Error AllErr = Error::success();
115
116 {
117 std::lock_guard<std::mutex> Lock(Mutex);
118
119 for (auto Base : llvm::reverse(C&: Bases)) {
120
121 if (Error Err = shared::runDeallocActions(
122 DAs: Allocations[Base].DeinitializationActions)) {
123 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
124 }
125
126 // Reset protections to read/write so the area can be reused
127 if (auto EC = sys::Memory::protectMappedMemory(
128 Block: {Base.toPtr<void *>(), Allocations[Base].Size},
129 Flags: sys::Memory::ProtectionFlags::MF_READ |
130 sys::Memory::ProtectionFlags::MF_WRITE)) {
131 AllErr = joinErrors(E1: std::move(AllErr), E2: errorCodeToError(EC));
132 }
133
134 Allocations.erase(Val: Base);
135 }
136 }
137
138 OnDeinitialized(std::move(AllErr));
139}
140
141void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
142 OnReleasedFunction OnReleased) {
143 Error Err = Error::success();
144
145 for (auto Base : Bases) {
146 std::vector<ExecutorAddr> AllocAddrs;
147 size_t Size;
148 {
149 std::lock_guard<std::mutex> Lock(Mutex);
150 auto &R = Reservations[Base.toPtr<void *>()];
151 Size = R.Size;
152 AllocAddrs.swap(x&: R.Allocations);
153 }
154
155 // deinitialize sub allocations
156 std::promise<MSVCPError> P;
157 auto F = P.get_future();
158 deinitialize(Bases: AllocAddrs, OnDeinitialized: [&](Error Err) { P.set_value(std::move(Err)); });
159 if (Error E = F.get()) {
160 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
161 }
162
163 // free the memory
164 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
165
166 auto EC = sys::Memory::releaseMappedMemory(Block&: MB);
167 if (EC) {
168 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
169 }
170
171 std::lock_guard<std::mutex> Lock(Mutex);
172 Reservations.erase(Val: Base.toPtr<void *>());
173 }
174
175 OnReleased(std::move(Err));
176}
177
178InProcessMemoryMapper::~InProcessMemoryMapper() {
179 std::vector<ExecutorAddr> ReservationAddrs;
180 {
181 std::lock_guard<std::mutex> Lock(Mutex);
182
183 ReservationAddrs.reserve(n: Reservations.size());
184 for (const auto &R : Reservations) {
185 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
186 }
187 }
188
189 std::promise<MSVCPError> P;
190 auto F = P.get_future();
191 release(Bases: ReservationAddrs, OnReleased: [&](Error Err) { P.set_value(std::move(Err)); });
192 cantFail(Err: F.get());
193}
194
195// SharedMemoryMapper
196
197SharedMemoryMapper::SharedMemoryMapper(ExecutionSession &ES,
198 SharedMemoryMapBindings B,
199 size_t PageSize)
200 : ES(ES), B(std::move(B)), PageSize(PageSize) {
201#if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
202 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
203#endif
204}
205
206Expected<std::unique_ptr<SharedMemoryMapper>>
207SharedMemoryMapper::Create(ExecutionSession &ES, SharedMemoryMapBindings B) {
208#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
209 auto PageSize = sys::Process::getPageSize();
210 if (!PageSize)
211 return PageSize.takeError();
212
213 return std::make_unique<SharedMemoryMapper>(args&: ES, args: std::move(B), args&: *PageSize);
214#else
215 return make_error<StringError>(
216 "SharedMemoryMapper is not supported on this platform yet",
217 inconvertibleErrorCode());
218#endif
219}
220
221void SharedMemoryMapper::reserve(size_t NumBytes,
222 OnReservedFunction OnReserved) {
223#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
224
225 int SharedMemoryId = -1;
226 B.Reserve(
227 [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId](
228 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
229 if (!Result)
230 return OnReserved(Result.takeError());
231
232 ExecutorAddr RemoteAddr;
233 std::string SharedMemoryName;
234 std::tie(args&: RemoteAddr, args&: SharedMemoryName) = std::move(*Result);
235
236 void *LocalAddr = nullptr;
237
238#if defined(LLVM_ON_UNIX)
239
240#if defined(__MVS__)
241 ArrayRef<uint8_t> Data(
242 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
243 SharedMemoryName.size());
244 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
245 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
246 SharedMemoryId =
247 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
248 if (SharedMemoryId < 0) {
249 return OnReserved(errorCodeToError(
250 std::error_code(errno, std::generic_category())));
251 }
252 LocalAddr = shmat(SharedMemoryId, nullptr, 0);
253 if (LocalAddr == reinterpret_cast<void *>(-1)) {
254 return OnReserved(errorCodeToError(
255 std::error_code(errno, std::generic_category())));
256 }
257#else
258 int SharedMemoryFile = shm_open(name: SharedMemoryName.c_str(), O_RDWR, mode: 0700);
259 if (SharedMemoryFile < 0) {
260 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
261 }
262
263 // this prevents other processes from accessing it by name
264 shm_unlink(name: SharedMemoryName.c_str());
265
266 LocalAddr = mmap(addr: nullptr, len: NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
267 fd: SharedMemoryFile, offset: 0);
268 if (LocalAddr == MAP_FAILED) {
269 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
270 }
271
272 close(fd: SharedMemoryFile);
273#endif
274
275#elif defined(_WIN32)
276
277 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
278 SharedMemoryName.end());
279 HANDLE SharedMemoryFile = OpenFileMappingW(
280 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
281 if (!SharedMemoryFile)
282 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
283
284 LocalAddr =
285 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
286 if (!LocalAddr) {
287 CloseHandle(SharedMemoryFile);
288 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
289 }
290
291 CloseHandle(SharedMemoryFile);
292
293#endif
294 {
295 std::lock_guard<std::mutex> Lock(Mutex);
296 Reservations.insert(
297 x: {RemoteAddr, {.LocalAddr: LocalAddr, .Size: NumBytes, .SharedMemoryId: SharedMemoryId}});
298 }
299
300 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
301 },
302 ES, B.Instance, static_cast<uint64_t>(NumBytes));
303
304#else
305 OnReserved(make_error<StringError>(
306 "SharedMemoryMapper is not supported on this platform yet",
307 inconvertibleErrorCode()));
308#endif
309}
310
311char *SharedMemoryMapper::prepare(jitlink::LinkGraph &G, ExecutorAddr Addr,
312 size_t ContentSize) {
313 auto R = Reservations.upper_bound(x: Addr);
314 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
315 R--;
316
317 ExecutorAddrDiff Offset = Addr - R->first;
318
319 return static_cast<char *>(R->second.LocalAddr) + Offset;
320}
321
322void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
323 OnInitializedFunction OnInitialized) {
324 auto Reservation = Reservations.upper_bound(x: AI.MappingBase);
325 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
326 Reservation--;
327
328 auto AllocationOffset = AI.MappingBase - Reservation->first;
329
330 tpctypes::SharedMemoryFinalizeRequest FR;
331
332 AI.Actions.swap(x&: FR.Actions);
333
334 FR.Segments.reserve(n: AI.Segments.size());
335
336 for (auto Segment : AI.Segments) {
337 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
338 AllocationOffset + Segment.Offset;
339 std::memset(s: Base + Segment.ContentSize, c: 0, n: Segment.ZeroFillSize);
340
341 tpctypes::SharedMemorySegFinalizeRequest SegReq;
342 SegReq.RAG = {Segment.AG.getMemProt(),
343 Segment.AG.getMemLifetime() == MemLifetime::Finalize};
344 SegReq.Addr = AI.MappingBase + Segment.Offset;
345 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
346
347 FR.Segments.push_back(x: SegReq);
348 }
349
350 B.Initialize(std::move(OnInitialized), ES, B.Instance, Reservation->first,
351 std::move(FR));
352}
353
354void SharedMemoryMapper::deinitialize(
355 ArrayRef<ExecutorAddr> Allocations,
356 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
357 B.Deinitialize(std::move(OnDeinitialized), ES, B.Instance, Allocations);
358}
359
360void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
361 OnReleasedFunction OnReleased) {
362#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
363 Error Err = Error::success();
364
365 {
366 std::lock_guard<std::mutex> Lock(Mutex);
367
368 for (auto Base : Bases) {
369
370#if defined(LLVM_ON_UNIX)
371
372#if defined(__MVS__)
373 if (shmdt(Reservations[Base].LocalAddr) < 0 ||
374 shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0)
375 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
376#else
377 if (munmap(addr: Reservations[Base].LocalAddr, len: Reservations[Base].Size) != 0)
378 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
379#endif
380
381#elif defined(_WIN32)
382
383 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
384 Err = joinErrors(std::move(Err),
385 errorCodeToError(mapWindowsError(GetLastError())));
386
387#endif
388
389 Reservations.erase(x: Base);
390 }
391 }
392
393 B.Release(
394 [OnReleased = std::move(OnReleased),
395 Err = std::move(Err)](Error Result) mutable {
396 return OnReleased(joinErrors(E1: std::move(Err), E2: std::move(Result)));
397 },
398 ES, B.Instance, Bases);
399#else
400 OnReleased(make_error<StringError>(
401 "SharedMemoryMapper is not supported on this platform yet",
402 inconvertibleErrorCode()));
403#endif
404}
405
406SharedMemoryMapper::~SharedMemoryMapper() {
407 std::lock_guard<std::mutex> Lock(Mutex);
408 for (const auto &R : Reservations) {
409
410#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
411
412#if defined(__MVS__)
413 shmdt(R.second.LocalAddr);
414#else
415 munmap(addr: R.second.LocalAddr, len: R.second.Size);
416#endif
417
418#elif defined(_WIN32)
419
420 UnmapViewOfFile(R.second.LocalAddr);
421
422#else
423
424 (void)R;
425
426#endif
427 }
428}
429
430} // namespace orc
431
432} // namespace llvm
433