1//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10
11#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/Support/WindowsError.h"
14
15#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
16#include <fcntl.h>
17#include <sys/mman.h>
18#if defined(__MVS__)
19#include "llvm/Support/BLAKE3.h"
20#include <sys/shm.h>
21#endif
22#include <unistd.h>
23#elif defined(_WIN32)
24#include <windows.h>
25#endif
26
27namespace llvm {
28namespace orc {
29
30MemoryMapper::~MemoryMapper() {}
31
32InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
33 : PageSize(PageSize) {}
34
35Expected<std::unique_ptr<InProcessMemoryMapper>>
36InProcessMemoryMapper::Create() {
37 auto PageSize = sys::Process::getPageSize();
38 if (!PageSize)
39 return PageSize.takeError();
40 return std::make_unique<InProcessMemoryMapper>(args&: *PageSize);
41}
42
43void InProcessMemoryMapper::reserve(size_t NumBytes,
44 OnReservedFunction OnReserved) {
45 std::error_code EC;
46 auto MB = sys::Memory::allocateMappedMemory(
47 NumBytes, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
48
49 if (EC)
50 return OnReserved(errorCodeToError(EC));
51
52 {
53 std::lock_guard<std::mutex> Lock(Mutex);
54 Reservations[MB.base()].Size = MB.allocatedSize();
55 }
56
57 OnReserved(
58 ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: MB.base()), MB.allocatedSize()));
59}
60
61char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
62 return Addr.toPtr<char *>();
63}
64
65void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
66 OnInitializedFunction OnInitialized) {
67 ExecutorAddr MinAddr(~0ULL);
68 ExecutorAddr MaxAddr(0);
69
70 // FIXME: Release finalize lifetime segments.
71 for (auto &Segment : AI.Segments) {
72 auto Base = AI.MappingBase + Segment.Offset;
73 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
74
75 if (Base < MinAddr)
76 MinAddr = Base;
77
78 if (Base + Size > MaxAddr)
79 MaxAddr = Base + Size;
80
81 std::memset(s: (Base + Segment.ContentSize).toPtr<void *>(), c: 0,
82 n: Segment.ZeroFillSize);
83
84 if (auto EC = sys::Memory::protectMappedMemory(
85 Block: {Base.toPtr<void *>(), Size},
86 Flags: toSysMemoryProtectionFlags(MP: Segment.AG.getMemProt()))) {
87 return OnInitialized(errorCodeToError(EC));
88 }
89 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
90 sys::Memory::InvalidateInstructionCache(Addr: Base.toPtr<void *>(), Len: Size);
91 }
92
93 std::vector<shared::WrapperFunctionCall> DeinitializeActions;
94 {
95 std::promise<MSVCPExpected<std::vector<shared::WrapperFunctionCall>>> P;
96 auto F = P.get_future();
97 shared::runFinalizeActions(
98 AAs&: AI.Actions, OnComplete: [&](Expected<std::vector<shared::WrapperFunctionCall>> R) {
99 P.set_value(std::move(R));
100 });
101 if (auto DeinitializeActionsOrErr = F.get())
102 DeinitializeActions = std::move(*DeinitializeActionsOrErr);
103 else
104 return OnInitialized(DeinitializeActionsOrErr.takeError());
105 }
106
107 {
108 std::lock_guard<std::mutex> Lock(Mutex);
109
110 // This is the maximum range whose permission have been possibly modified
111 auto &Alloc = Allocations[MinAddr];
112 Alloc.Size = MaxAddr - MinAddr;
113 Alloc.DeinitializationActions = std::move(DeinitializeActions);
114 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(x: MinAddr);
115 }
116
117 OnInitialized(MinAddr);
118}
119
120void InProcessMemoryMapper::deinitialize(
121 ArrayRef<ExecutorAddr> Bases,
122 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
123 Error AllErr = Error::success();
124
125 {
126 std::lock_guard<std::mutex> Lock(Mutex);
127
128 for (auto Base : llvm::reverse(C&: Bases)) {
129
130 shared::runDeallocActions(
131 DAs: Allocations[Base].DeinitializationActions, OnComplete: [&](Error Err) {
132 AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err));
133 });
134
135 // Reset protections to read/write so the area can be reused
136 if (auto EC = sys::Memory::protectMappedMemory(
137 Block: {Base.toPtr<void *>(), Allocations[Base].Size},
138 Flags: sys::Memory::ProtectionFlags::MF_READ |
139 sys::Memory::ProtectionFlags::MF_WRITE)) {
140 AllErr = joinErrors(E1: std::move(AllErr), E2: errorCodeToError(EC));
141 }
142
143 Allocations.erase(Val: Base);
144 }
145 }
146
147 OnDeinitialized(std::move(AllErr));
148}
149
150void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
151 OnReleasedFunction OnReleased) {
152 Error Err = Error::success();
153
154 for (auto Base : Bases) {
155 std::vector<ExecutorAddr> AllocAddrs;
156 size_t Size;
157 {
158 std::lock_guard<std::mutex> Lock(Mutex);
159 auto &R = Reservations[Base.toPtr<void *>()];
160 Size = R.Size;
161 AllocAddrs.swap(x&: R.Allocations);
162 }
163
164 // deinitialize sub allocations
165 std::promise<MSVCPError> P;
166 auto F = P.get_future();
167 deinitialize(Bases: AllocAddrs, OnDeinitialized: [&](Error Err) { P.set_value(std::move(Err)); });
168 if (Error E = F.get()) {
169 Err = joinErrors(E1: std::move(Err), E2: std::move(E));
170 }
171
172 // free the memory
173 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
174
175 auto EC = sys::Memory::releaseMappedMemory(Block&: MB);
176 if (EC) {
177 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
178 }
179
180 std::lock_guard<std::mutex> Lock(Mutex);
181 Reservations.erase(Val: Base.toPtr<void *>());
182 }
183
184 OnReleased(std::move(Err));
185}
186
187InProcessMemoryMapper::~InProcessMemoryMapper() {
188 std::vector<ExecutorAddr> ReservationAddrs;
189 {
190 std::lock_guard<std::mutex> Lock(Mutex);
191
192 ReservationAddrs.reserve(n: Reservations.size());
193 for (const auto &R : Reservations) {
194 ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst()));
195 }
196 }
197
198 std::promise<MSVCPError> P;
199 auto F = P.get_future();
200 release(Bases: ReservationAddrs, OnReleased: [&](Error Err) { P.set_value(std::move(Err)); });
201 cantFail(Err: F.get());
202}
203
204// SharedMemoryMapper
205
206SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
207 SymbolAddrs SAs, size_t PageSize)
208 : EPC(EPC), SAs(SAs), PageSize(PageSize) {
209#if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
210 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
211#endif
212}
213
214Expected<std::unique_ptr<SharedMemoryMapper>>
215SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
216#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
217 auto PageSize = sys::Process::getPageSize();
218 if (!PageSize)
219 return PageSize.takeError();
220
221 return std::make_unique<SharedMemoryMapper>(args&: EPC, args&: SAs, args&: *PageSize);
222#else
223 return make_error<StringError>(
224 "SharedMemoryMapper is not supported on this platform yet",
225 inconvertibleErrorCode());
226#endif
227}
228
229void SharedMemoryMapper::reserve(size_t NumBytes,
230 OnReservedFunction OnReserved) {
231#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
232
233 int SharedMemoryId = -1;
234 EPC.callSPSWrapperAsync<
235 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
236 WrapperFnAddr: SAs.Reserve,
237 SendResult: [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId](
238 Error SerializationErr,
239 Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
240 if (SerializationErr) {
241 cantFail(Err: Result.takeError());
242 return OnReserved(std::move(SerializationErr));
243 }
244
245 if (!Result)
246 return OnReserved(Result.takeError());
247
248 ExecutorAddr RemoteAddr;
249 std::string SharedMemoryName;
250 std::tie(args&: RemoteAddr, args&: SharedMemoryName) = std::move(*Result);
251
252 void *LocalAddr = nullptr;
253
254#if defined(LLVM_ON_UNIX)
255
256#if defined(__MVS__)
257 ArrayRef<uint8_t> Data(
258 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
259 SharedMemoryName.size());
260 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
261 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
262 SharedMemoryId =
263 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
264 if (SharedMemoryId < 0) {
265 return OnReserved(errorCodeToError(
266 std::error_code(errno, std::generic_category())));
267 }
268 LocalAddr = shmat(SharedMemoryId, nullptr, 0);
269 if (LocalAddr == reinterpret_cast<void *>(-1)) {
270 return OnReserved(errorCodeToError(
271 std::error_code(errno, std::generic_category())));
272 }
273#else
274 int SharedMemoryFile = shm_open(name: SharedMemoryName.c_str(), O_RDWR, mode: 0700);
275 if (SharedMemoryFile < 0) {
276 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
277 }
278
279 // this prevents other processes from accessing it by name
280 shm_unlink(name: SharedMemoryName.c_str());
281
282 LocalAddr = mmap(addr: nullptr, len: NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
283 fd: SharedMemoryFile, offset: 0);
284 if (LocalAddr == MAP_FAILED) {
285 return OnReserved(errorCodeToError(EC: errnoAsErrorCode()));
286 }
287
288 close(fd: SharedMemoryFile);
289#endif
290
291#elif defined(_WIN32)
292
293 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
294 SharedMemoryName.end());
295 HANDLE SharedMemoryFile = OpenFileMappingW(
296 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
297 if (!SharedMemoryFile)
298 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
299
300 LocalAddr =
301 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
302 if (!LocalAddr) {
303 CloseHandle(SharedMemoryFile);
304 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
305 }
306
307 CloseHandle(SharedMemoryFile);
308
309#endif
310 {
311 std::lock_guard<std::mutex> Lock(Mutex);
312 Reservations.insert(
313 x: {RemoteAddr, {.LocalAddr: LocalAddr, .Size: NumBytes, .SharedMemoryId: SharedMemoryId}});
314 }
315
316 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
317 },
318 Args: SAs.Instance, Args: static_cast<uint64_t>(NumBytes));
319
320#else
321 OnReserved(make_error<StringError>(
322 "SharedMemoryMapper is not supported on this platform yet",
323 inconvertibleErrorCode()));
324#endif
325}
326
327char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
328 auto R = Reservations.upper_bound(x: Addr);
329 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
330 R--;
331
332 ExecutorAddrDiff Offset = Addr - R->first;
333
334 return static_cast<char *>(R->second.LocalAddr) + Offset;
335}
336
337void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
338 OnInitializedFunction OnInitialized) {
339 auto Reservation = Reservations.upper_bound(x: AI.MappingBase);
340 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
341 Reservation--;
342
343 auto AllocationOffset = AI.MappingBase - Reservation->first;
344
345 tpctypes::SharedMemoryFinalizeRequest FR;
346
347 AI.Actions.swap(x&: FR.Actions);
348
349 FR.Segments.reserve(n: AI.Segments.size());
350
351 for (auto Segment : AI.Segments) {
352 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
353 AllocationOffset + Segment.Offset;
354 std::memset(s: Base + Segment.ContentSize, c: 0, n: Segment.ZeroFillSize);
355
356 tpctypes::SharedMemorySegFinalizeRequest SegReq;
357 SegReq.RAG = {Segment.AG.getMemProt(),
358 Segment.AG.getMemLifetime() == MemLifetime::Finalize};
359 SegReq.Addr = AI.MappingBase + Segment.Offset;
360 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
361
362 FR.Segments.push_back(x: SegReq);
363 }
364
365 EPC.callSPSWrapperAsync<
366 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
367 WrapperFnAddr: SAs.Initialize,
368 SendResult: [OnInitialized = std::move(OnInitialized)](
369 Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
370 if (SerializationErr) {
371 cantFail(Err: Result.takeError());
372 return OnInitialized(std::move(SerializationErr));
373 }
374
375 OnInitialized(std::move(Result));
376 },
377 Args: SAs.Instance, Args: Reservation->first, Args: std::move(FR));
378}
379
380void SharedMemoryMapper::deinitialize(
381 ArrayRef<ExecutorAddr> Allocations,
382 MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
383 EPC.callSPSWrapperAsync<
384 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
385 WrapperFnAddr: SAs.Deinitialize,
386 SendResult: [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
387 Error Result) mutable {
388 if (SerializationErr) {
389 cantFail(Err: std::move(Result));
390 return OnDeinitialized(std::move(SerializationErr));
391 }
392
393 OnDeinitialized(std::move(Result));
394 },
395 Args: SAs.Instance, Args: Allocations);
396}
397
398void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
399 OnReleasedFunction OnReleased) {
400#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
401 Error Err = Error::success();
402
403 {
404 std::lock_guard<std::mutex> Lock(Mutex);
405
406 for (auto Base : Bases) {
407
408#if defined(LLVM_ON_UNIX)
409
410#if defined(__MVS__)
411 if (shmdt(Reservations[Base].LocalAddr) < 0 ||
412 shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0)
413 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
414#else
415 if (munmap(addr: Reservations[Base].LocalAddr, len: Reservations[Base].Size) != 0)
416 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode()));
417#endif
418
419#elif defined(_WIN32)
420
421 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
422 Err = joinErrors(std::move(Err),
423 errorCodeToError(mapWindowsError(GetLastError())));
424
425#endif
426
427 Reservations.erase(x: Base);
428 }
429 }
430
431 EPC.callSPSWrapperAsync<
432 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
433 WrapperFnAddr: SAs.Release,
434 SendResult: [OnReleased = std::move(OnReleased),
435 Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
436 if (SerializationErr) {
437 cantFail(Err: std::move(Result));
438 return OnReleased(
439 joinErrors(E1: std::move(Err), E2: std::move(SerializationErr)));
440 }
441
442 return OnReleased(joinErrors(E1: std::move(Err), E2: std::move(Result)));
443 },
444 Args: SAs.Instance, Args: Bases);
445#else
446 OnReleased(make_error<StringError>(
447 "SharedMemoryMapper is not supported on this platform yet",
448 inconvertibleErrorCode()));
449#endif
450}
451
452SharedMemoryMapper::~SharedMemoryMapper() {
453 std::lock_guard<std::mutex> Lock(Mutex);
454 for (const auto &R : Reservations) {
455
456#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
457
458#if defined(__MVS__)
459 shmdt(R.second.LocalAddr);
460#else
461 munmap(addr: R.second.LocalAddr, len: R.second.Size);
462#endif
463
464#elif defined(_WIN32)
465
466 UnmapViewOfFile(R.second.LocalAddr);
467
468#else
469
470 (void)R;
471
472#endif
473 }
474}
475
476} // namespace orc
477
478} // namespace llvm
479