1//===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
10
11#include "llvm/ADT/ScopeExit.h"
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/Support/FormatVariadic.h"
14
15#define DEBUG_TYPE "orc"
16
17namespace llvm {
18namespace orc {
19namespace rt_bootstrap {
20
21SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
22 assert(Slabs.empty() && "shutdown not called?");
23}
24
25Expected<ExecutorAddr> SimpleExecutorMemoryManager::reserve(uint64_t Size) {
26 std::error_code EC;
27 auto MB = sys::Memory::allocateMappedMemory(
28 NumBytes: Size, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
29 if (EC)
30 return errorCodeToError(EC);
31 std::lock_guard<std::mutex> Lock(M);
32 assert(!Slabs.count(MB.base()) && "Duplicate allocation addr");
33 Slabs[MB.base()].Size = Size;
34 return ExecutorAddr::fromPtr(Ptr: MB.base());
35}
36
37Expected<ExecutorAddr>
38SimpleExecutorMemoryManager::initialize(tpctypes::FinalizeRequest &FR) {
39 if (FR.Segments.empty()) {
40 if (FR.Actions.empty())
41 return make_error<StringError>(Args: "Finalization request is empty",
42 Args: inconvertibleErrorCode());
43 else
44 return make_error<StringError>(Args: "Finalization actions attached to empty "
45 "finalization request",
46 Args: inconvertibleErrorCode());
47 }
48
49 ExecutorAddrRange RR(FR.Segments.front().Addr, FR.Segments.front().Addr);
50
51 std::vector<sys::MemoryBlock> MBsToReset;
52 llvm::scope_exit ResetMBs([&]() {
53 for (auto &MB : MBsToReset)
54 sys::Memory::protectMappedMemory(Block: MB, Flags: sys::Memory::MF_READ |
55 sys::Memory::MF_WRITE);
56 sys::Memory::InvalidateInstructionCache(Addr: RR.Start.toPtr<void *>(),
57 Len: RR.size());
58 });
59
60 // Copy content and apply permissions.
61 for (auto &Seg : FR.Segments) {
62 RR.Start = std::min(a: RR.Start, b: Seg.Addr);
63 RR.End = std::max(a: RR.End, b: Seg.Addr + Seg.Size);
64
65 // Check segment ranges.
66 if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
67 return make_error<StringError>(
68 Args: formatv(Fmt: "Segment {0:x} content size ({1:x} bytes) "
69 "exceeds segment size ({2:x} bytes)",
70 Vals: Seg.Addr.getValue(), Vals: Seg.Content.size(), Vals&: Seg.Size),
71 Args: inconvertibleErrorCode());
72 ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
73 if (LLVM_UNLIKELY(Seg.Addr < RR.Start || SegEnd > RR.End))
74 return make_error<StringError>(
75 Args: formatv(Fmt: "Segment {0:x} -- {1:x} crosses boundary of "
76 "allocation {2:x} -- {3:x}",
77 Vals&: Seg.Addr, Vals&: SegEnd, Vals&: RR.Start, Vals&: RR.End),
78 Args: inconvertibleErrorCode());
79
80 char *Mem = Seg.Addr.toPtr<char *>();
81 if (!Seg.Content.empty())
82 memcpy(dest: Mem, src: Seg.Content.data(), n: Seg.Content.size());
83 memset(s: Mem + Seg.Content.size(), c: 0, n: Seg.Size - Seg.Content.size());
84 assert(Seg.Size <= std::numeric_limits<size_t>::max());
85
86 sys::MemoryBlock MB(Mem, Seg.Size);
87 if (auto EC = sys::Memory::protectMappedMemory(
88 Block: MB, Flags: toSysMemoryProtectionFlags(MP: Seg.RAG.Prot)))
89 return errorCodeToError(EC);
90
91 MBsToReset.push_back(x: MB);
92
93 if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec)
94 sys::Memory::InvalidateInstructionCache(Addr: Mem, Len: Seg.Size);
95 }
96
97 auto DeallocActions = runFinalizeActions(AAs&: FR.Actions);
98 if (!DeallocActions)
99 return DeallocActions.takeError();
100
101 {
102 std::lock_guard<std::mutex> Lock(M);
103 auto Region = createRegionInfo(R: RR, Context: "In initialize");
104 if (!Region)
105 return Region.takeError();
106 Region->DeallocActions = std::move(*DeallocActions);
107 }
108
109 // Successful initialization.
110 ResetMBs.release();
111
112 return RR.Start;
113}
114
115Error SimpleExecutorMemoryManager::deinitialize(
116 const std::vector<ExecutorAddr> &InitKeys) {
117 Error Err = Error::success();
118
119 for (auto &KeyAddr : llvm::reverse(C: InitKeys)) {
120 std::vector<shared::WrapperFunctionCall> DeallocActions;
121 {
122 std::scoped_lock<std::mutex> Lock(M);
123 auto Slab = getSlabInfo(A: KeyAddr, Context: "In deinitialize");
124 if (!Slab) {
125 Err = joinErrors(E1: std::move(Err), E2: Slab.takeError());
126 continue;
127 }
128
129 auto RI = getRegionInfo(Slab&: *Slab, A: KeyAddr, Context: "In deinitialize");
130 if (!RI) {
131 Err = joinErrors(E1: std::move(Err), E2: RI.takeError());
132 continue;
133 }
134
135 DeallocActions = std::move(RI->DeallocActions);
136 }
137
138 Err = joinErrors(E1: std::move(Err),
139 E2: runDeallocActions(DAs: std::move(DeallocActions)));
140 }
141
142 return Err;
143}
144
145Error SimpleExecutorMemoryManager::release(
146 const std::vector<ExecutorAddr> &Bases) {
147 Error Err = Error::success();
148
149 // TODO: Prohibit new initializations within the slabs being removed?
150 for (auto &Base : llvm::reverse(C: Bases)) {
151 std::vector<shared::WrapperFunctionCall> DeallocActions;
152 sys::MemoryBlock MB;
153
154 {
155 std::scoped_lock<std::mutex> Lock(M);
156
157 auto SlabI = Slabs.find(x: Base.toPtr<void *>());
158 if (SlabI == Slabs.end()) {
159 Err = joinErrors(
160 E1: std::move(Err),
161 E2: make_error<StringError>(Args: "In release, " + formatv(Fmt: "{0:x}", Vals: Base) +
162 " is not part of any reserved "
163 "address range",
164 Args: inconvertibleErrorCode()));
165 continue;
166 }
167
168 auto &Slab = SlabI->second;
169
170 for (auto &[Addr, Region] : Slab.Regions)
171 llvm::copy(Range&: Region.DeallocActions, Out: back_inserter(x&: DeallocActions));
172
173 MB = {Base.toPtr<void *>(), Slab.Size};
174
175 Slabs.erase(position: SlabI);
176 }
177
178 Err = joinErrors(E1: std::move(Err), E2: runDeallocActions(DAs: DeallocActions));
179 if (auto EC = sys::Memory::releaseMappedMemory(Block&: MB))
180 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
181 }
182
183 return Err;
184}
185
186Error SimpleExecutorMemoryManager::shutdown() {
187
188 // TODO: Prevent new allocations during shutdown.
189 std::vector<ExecutorAddr> Bases;
190 {
191 std::scoped_lock<std::mutex> Lock(M);
192 for (auto &[Base, Slab] : Slabs)
193 Bases.push_back(x: ExecutorAddr::fromPtr(Ptr: Base));
194 }
195
196 return release(Bases);
197}
198
199void SimpleExecutorMemoryManager::addBootstrapSymbols(
200 StringMap<ExecutorAddr> &M) {
201 M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(Ptr: this);
202 M[rt::SimpleExecutorMemoryManagerReserveWrapperName] =
203 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
204 M[rt::SimpleExecutorMemoryManagerInitializeWrapperName] =
205 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
206 M[rt::SimpleExecutorMemoryManagerDeinitializeWrapperName] =
207 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
208 M[rt::SimpleExecutorMemoryManagerReleaseWrapperName] =
209 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
210
211 {
212 // Also provide SimpleNativeMemoryMap symbols for compatibility.
213 // FIXME: We should codify a "simple" memory manager interface and make
214 // SimpleExecutorMemoryManager its LLVM-based implementation, and
215 // SimpleNativeMemoryMap its ORC-runtime implementation.
216 const auto &SNs = rt::orc_rt_SimpleNativeMemoryMapSPSSymbols;
217 M[SNs.AllocatorName] = ExecutorAddr::fromPtr(Ptr: this);
218 M[SNs.ReserveName] = ExecutorAddr::fromPtr(Ptr: reserveWrapper);
219 M[SNs.InitializeName] = ExecutorAddr::fromPtr(Ptr: initializeWrapper);
220 M[SNs.DeinitializeName] = ExecutorAddr::fromPtr(Ptr: deinitializeWrapper);
221 M[SNs.ReleaseName] = ExecutorAddr::fromPtr(Ptr: releaseWrapper);
222 }
223}
224
225Expected<SimpleExecutorMemoryManager::SlabInfo &>
226SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddr A, StringRef Context) {
227 auto MakeBadSlabError = [&]() {
228 return make_error<StringError>(
229 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
230 " is not part of any reserved address range",
231 Args: inconvertibleErrorCode());
232 };
233
234 auto I = Slabs.upper_bound(x: A.toPtr<void *>());
235 if (I == Slabs.begin())
236 return MakeBadSlabError();
237 --I;
238 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
239 .contains(Addr: A))
240 return MakeBadSlabError();
241
242 return I->second;
243}
244
245Expected<SimpleExecutorMemoryManager::SlabInfo &>
246SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddrRange R,
247 StringRef Context) {
248 auto MakeBadSlabError = [&]() {
249 return make_error<StringError>(
250 Args: Context + ", range " + formatv(Fmt: "{0:x}", Vals&: R) +
251 " is not part of any reserved address range",
252 Args: inconvertibleErrorCode());
253 };
254
255 auto I = Slabs.upper_bound(x: R.Start.toPtr<void *>());
256 if (I == Slabs.begin())
257 return MakeBadSlabError();
258 --I;
259 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
260 .contains(Other: R))
261 return MakeBadSlabError();
262
263 return I->second;
264}
265
266Expected<SimpleExecutorMemoryManager::RegionInfo &>
267SimpleExecutorMemoryManager::createRegionInfo(ExecutorAddrRange R,
268 StringRef Context) {
269
270 auto Slab = getSlabInfo(R, Context);
271 if (!Slab)
272 return Slab.takeError();
273
274 auto MakeBadRegionError = [&](ExecutorAddrRange Other, bool Prev) {
275 return make_error<StringError>(Args: Context + ", region " + formatv(Fmt: "{0:x}", Vals&: R) +
276 " overlaps " +
277 (Prev ? "previous" : "following") +
278 " region " + formatv(Fmt: "{0:x}", Vals&: Other),
279 Args: inconvertibleErrorCode());
280 };
281
282 auto I = Slab->Regions.upper_bound(x: R.Start);
283 if (I != Slab->Regions.begin()) {
284 auto J = std::prev(x: I);
285 ExecutorAddrRange PrevRange(J->first, J->second.Size);
286 if (PrevRange.overlaps(Other: R))
287 return MakeBadRegionError(PrevRange, true);
288 }
289 if (I != Slab->Regions.end()) {
290 ExecutorAddrRange NextRange(I->first, I->second.Size);
291 if (NextRange.overlaps(Other: R))
292 return MakeBadRegionError(NextRange, false);
293 }
294
295 auto &RInfo = Slab->Regions[R.Start];
296 RInfo.Size = R.size();
297 return RInfo;
298}
299
300Expected<SimpleExecutorMemoryManager::RegionInfo &>
301SimpleExecutorMemoryManager::getRegionInfo(SlabInfo &Slab, ExecutorAddr A,
302 StringRef Context) {
303 auto I = Slab.Regions.find(x: A);
304 if (I == Slab.Regions.end())
305 return make_error<StringError>(
306 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
307 " does not correspond to the start of any initialized region",
308 Args: inconvertibleErrorCode());
309
310 return I->second;
311}
312
313Expected<SimpleExecutorMemoryManager::RegionInfo &>
314SimpleExecutorMemoryManager::getRegionInfo(ExecutorAddr A, StringRef Context) {
315 auto Slab = getSlabInfo(A, Context);
316 if (!Slab)
317 return Slab.takeError();
318
319 return getRegionInfo(Slab&: *Slab, A, Context);
320}
321
322llvm::orc::shared::CWrapperFunctionBuffer
323SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
324 size_t ArgSize) {
325 return shared::WrapperFunction<rt::SPSSimpleRemoteMemoryMapReserveSignature>::
326 handle(ArgData, ArgSize,
327 Handler: shared::makeMethodWrapperHandler(
328 Method: &SimpleExecutorMemoryManager::reserve))
329 .release();
330}
331
332llvm::orc::shared::CWrapperFunctionBuffer
333SimpleExecutorMemoryManager::initializeWrapper(const char *ArgData,
334 size_t ArgSize) {
335 return shared::
336 WrapperFunction<rt::SPSSimpleRemoteMemoryMapInitializeSignature>::handle(
337 ArgData, ArgSize,
338 Handler: shared::makeMethodWrapperHandler(
339 Method: &SimpleExecutorMemoryManager::initialize))
340 .release();
341}
342
343llvm::orc::shared::CWrapperFunctionBuffer
344SimpleExecutorMemoryManager::deinitializeWrapper(const char *ArgData,
345 size_t ArgSize) {
346 return shared::WrapperFunction<
347 rt::SPSSimpleRemoteMemoryMapDeinitializeSignature>::
348 handle(ArgData, ArgSize,
349 Handler: shared::makeMethodWrapperHandler(
350 Method: &SimpleExecutorMemoryManager::deinitialize))
351 .release();
352}
353
354llvm::orc::shared::CWrapperFunctionBuffer
355SimpleExecutorMemoryManager::releaseWrapper(const char *ArgData,
356 size_t ArgSize) {
357 return shared::WrapperFunction<rt::SPSSimpleRemoteMemoryMapReleaseSignature>::
358 handle(ArgData, ArgSize,
359 Handler: shared::makeMethodWrapperHandler(
360 Method: &SimpleExecutorMemoryManager::release))
361 .release();
362}
363
364} // namespace rt_bootstrap
365} // end namespace orc
366} // end namespace llvm
367