1//===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
10
11#include "llvm/ADT/ScopeExit.h"
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/ExecutionEngine/Orc/Shared/SPSCI/SimpleNativeMemoryMapSPSCI.h"
14#include "llvm/Support/FormatVariadic.h"
15
16#define DEBUG_TYPE "orc"
17
18namespace llvm {
19namespace orc {
20namespace rt_bootstrap {
21
22SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
23 assert(Slabs.empty() && "shutdown not called?");
24}
25
26Expected<ExecutorAddr> SimpleExecutorMemoryManager::reserve(uint64_t Size) {
27 std::error_code EC;
28 auto MB = sys::Memory::allocateMappedMemory(
29 NumBytes: Size, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
30 if (EC)
31 return errorCodeToError(EC);
32 std::lock_guard<std::mutex> Lock(M);
33 assert(!Slabs.count(MB.base()) && "Duplicate allocation addr");
34 Slabs[MB.base()].Size = Size;
35 return ExecutorAddr::fromPtr(Ptr: MB.base());
36}
37
38Expected<ExecutorAddr>
39SimpleExecutorMemoryManager::initialize(tpctypes::FinalizeRequest &FR) {
40 if (FR.Segments.empty()) {
41 if (FR.Actions.empty())
42 return make_error<StringError>(Args: "Finalization request is empty",
43 Args: inconvertibleErrorCode());
44 else
45 return make_error<StringError>(Args: "Finalization actions attached to empty "
46 "finalization request",
47 Args: inconvertibleErrorCode());
48 }
49
50 ExecutorAddrRange RR(FR.Segments.front().Addr, FR.Segments.front().Addr);
51
52 std::vector<sys::MemoryBlock> MBsToReset;
53 llvm::scope_exit ResetMBs([&]() {
54 for (auto &MB : MBsToReset)
55 sys::Memory::protectMappedMemory(Block: MB, Flags: sys::Memory::MF_READ |
56 sys::Memory::MF_WRITE);
57 sys::Memory::InvalidateInstructionCache(Addr: RR.Start.toPtr<void *>(),
58 Len: RR.size());
59 });
60
61 // Copy content and apply permissions.
62 for (auto &Seg : FR.Segments) {
63 RR.Start = std::min(a: RR.Start, b: Seg.Addr);
64 RR.End = std::max(a: RR.End, b: Seg.Addr + Seg.Size);
65
66 // Check segment ranges.
67 if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
68 return make_error<StringError>(
69 Args: formatv(Fmt: "Segment {0:x} content size ({1:x} bytes) "
70 "exceeds segment size ({2:x} bytes)",
71 Vals: Seg.Addr.getValue(), Vals: Seg.Content.size(), Vals&: Seg.Size),
72 Args: inconvertibleErrorCode());
73 ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
74 if (LLVM_UNLIKELY(Seg.Addr < RR.Start || SegEnd > RR.End))
75 return make_error<StringError>(
76 Args: formatv(Fmt: "Segment {0:x} -- {1:x} crosses boundary of "
77 "allocation {2:x} -- {3:x}",
78 Vals&: Seg.Addr, Vals&: SegEnd, Vals&: RR.Start, Vals&: RR.End),
79 Args: inconvertibleErrorCode());
80
81 char *Mem = Seg.Addr.toPtr<char *>();
82 if (!Seg.Content.empty())
83 memcpy(dest: Mem, src: Seg.Content.data(), n: Seg.Content.size());
84 memset(s: Mem + Seg.Content.size(), c: 0, n: Seg.Size - Seg.Content.size());
85 assert(Seg.Size <= std::numeric_limits<size_t>::max());
86
87 sys::MemoryBlock MB(Mem, Seg.Size);
88 if (auto EC = sys::Memory::protectMappedMemory(
89 Block: MB, Flags: toSysMemoryProtectionFlags(MP: Seg.RAG.Prot)))
90 return errorCodeToError(EC);
91
92 MBsToReset.push_back(x: MB);
93
94 if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec)
95 sys::Memory::InvalidateInstructionCache(Addr: Mem, Len: Seg.Size);
96 }
97
98 auto DeallocActions = runFinalizeActions(AAs&: FR.Actions);
99 if (!DeallocActions)
100 return DeallocActions.takeError();
101
102 {
103 std::lock_guard<std::mutex> Lock(M);
104 auto Region = createRegionInfo(R: RR, Context: "In initialize");
105 if (!Region)
106 return Region.takeError();
107 Region->DeallocActions = std::move(*DeallocActions);
108 }
109
110 // Successful initialization.
111 ResetMBs.release();
112
113 return RR.Start;
114}
115
116Error SimpleExecutorMemoryManager::deinitialize(
117 const std::vector<ExecutorAddr> &InitKeys) {
118 Error Err = Error::success();
119
120 for (auto &KeyAddr : llvm::reverse(C: InitKeys)) {
121 std::vector<shared::WrapperFunctionCall> DeallocActions;
122 {
123 std::scoped_lock<std::mutex> Lock(M);
124 auto Slab = getSlabInfo(A: KeyAddr, Context: "In deinitialize");
125 if (!Slab) {
126 Err = joinErrors(E1: std::move(Err), E2: Slab.takeError());
127 continue;
128 }
129
130 auto RI = getRegionInfo(Slab&: *Slab, A: KeyAddr, Context: "In deinitialize");
131 if (!RI) {
132 Err = joinErrors(E1: std::move(Err), E2: RI.takeError());
133 continue;
134 }
135
136 DeallocActions = std::move(RI->DeallocActions);
137 }
138
139 Err = joinErrors(E1: std::move(Err),
140 E2: runDeallocActions(DAs: std::move(DeallocActions)));
141 }
142
143 return Err;
144}
145
146Error SimpleExecutorMemoryManager::release(
147 const std::vector<ExecutorAddr> &Bases) {
148 Error Err = Error::success();
149
150 // TODO: Prohibit new initializations within the slabs being removed?
151 for (auto &Base : llvm::reverse(C: Bases)) {
152 std::vector<shared::WrapperFunctionCall> DeallocActions;
153 sys::MemoryBlock MB;
154
155 {
156 std::scoped_lock<std::mutex> Lock(M);
157
158 auto SlabI = Slabs.find(x: Base.toPtr<void *>());
159 if (SlabI == Slabs.end()) {
160 Err = joinErrors(
161 E1: std::move(Err),
162 E2: make_error<StringError>(Args: "In release, " + formatv(Fmt: "{0:x}", Vals: Base) +
163 " is not part of any reserved "
164 "address range",
165 Args: inconvertibleErrorCode()));
166 continue;
167 }
168
169 auto &Slab = SlabI->second;
170
171 for (auto &[Addr, Region] : Slab.Regions)
172 llvm::copy(Range&: Region.DeallocActions, Out: back_inserter(x&: DeallocActions));
173
174 MB = {Base.toPtr<void *>(), Slab.Size};
175
176 Slabs.erase(position: SlabI);
177 }
178
179 Err = joinErrors(E1: std::move(Err), E2: runDeallocActions(DAs: DeallocActions));
180 if (auto EC = sys::Memory::releaseMappedMemory(Block&: MB))
181 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
182 }
183
184 return Err;
185}
186
187Error SimpleExecutorMemoryManager::shutdown() {
188
189 // TODO: Prevent new allocations during shutdown.
190 std::vector<ExecutorAddr> Bases;
191 {
192 std::scoped_lock<std::mutex> Lock(M);
193 for (auto &[Base, Slab] : Slabs)
194 Bases.push_back(x: ExecutorAddr::fromPtr(Ptr: Base));
195 }
196
197 return release(Bases);
198}
199
200void SimpleExecutorMemoryManager::addBootstrapSymbols(
201 StringMap<ExecutorAddr> &M) {
202 M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(Ptr: this);
203 M[rt::SimpleExecutorMemoryManagerReserveWrapperName] =
204 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
205 M[rt::SimpleExecutorMemoryManagerInitializeWrapperName] =
206 ExecutorAddr::fromPtr(Ptr: &initializeWrapper);
207 M[rt::SimpleExecutorMemoryManagerDeinitializeWrapperName] =
208 ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper);
209 M[rt::SimpleExecutorMemoryManagerReleaseWrapperName] =
210 ExecutorAddr::fromPtr(Ptr: &releaseWrapper);
211
212 {
213 // Also provide SimpleNativeMemoryMap symbols for compatibility.
214 // FIXME: We should codify a "simple" memory manager interface and make
215 // SimpleExecutorMemoryManager its LLVM-based implementation, and
216 // SimpleNativeMemoryMap its ORC-runtime implementation.
217 namespace sps_ci = rt::sps_ci;
218 M[sps_ci::SimpleNativeMemoryMapInstanceName] = ExecutorAddr::fromPtr(Ptr: this);
219 M[sps_ci::MemMgrReserve::Name] = ExecutorAddr::fromPtr(Ptr: reserveWrapper);
220 M[sps_ci::MemMgrInitialize::Name] =
221 ExecutorAddr::fromPtr(Ptr: initializeWrapper);
222 M[sps_ci::MemMgrDeinitialize::Name] =
223 ExecutorAddr::fromPtr(Ptr: deinitializeWrapper);
224 M[sps_ci::MemMgrRelease::Name] = ExecutorAddr::fromPtr(Ptr: releaseWrapper);
225 }
226}
227
228Expected<SimpleExecutorMemoryManager::SlabInfo &>
229SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddr A, StringRef Context) {
230 auto MakeBadSlabError = [&]() {
231 return make_error<StringError>(
232 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
233 " is not part of any reserved address range",
234 Args: inconvertibleErrorCode());
235 };
236
237 auto I = Slabs.upper_bound(x: A.toPtr<void *>());
238 if (I == Slabs.begin())
239 return MakeBadSlabError();
240 --I;
241 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
242 .contains(Addr: A))
243 return MakeBadSlabError();
244
245 return I->second;
246}
247
248Expected<SimpleExecutorMemoryManager::SlabInfo &>
249SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddrRange R,
250 StringRef Context) {
251 auto MakeBadSlabError = [&]() {
252 return make_error<StringError>(
253 Args: Context + ", range " + formatv(Fmt: "{0:x}", Vals&: R) +
254 " is not part of any reserved address range",
255 Args: inconvertibleErrorCode());
256 };
257
258 auto I = Slabs.upper_bound(x: R.Start.toPtr<void *>());
259 if (I == Slabs.begin())
260 return MakeBadSlabError();
261 --I;
262 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
263 .contains(Other: R))
264 return MakeBadSlabError();
265
266 return I->second;
267}
268
269Expected<SimpleExecutorMemoryManager::RegionInfo &>
270SimpleExecutorMemoryManager::createRegionInfo(ExecutorAddrRange R,
271 StringRef Context) {
272
273 auto Slab = getSlabInfo(R, Context);
274 if (!Slab)
275 return Slab.takeError();
276
277 auto MakeBadRegionError = [&](ExecutorAddrRange Other, bool Prev) {
278 return make_error<StringError>(Args: Context + ", region " + formatv(Fmt: "{0:x}", Vals&: R) +
279 " overlaps " +
280 (Prev ? "previous" : "following") +
281 " region " + formatv(Fmt: "{0:x}", Vals&: Other),
282 Args: inconvertibleErrorCode());
283 };
284
285 auto I = Slab->Regions.upper_bound(x: R.Start);
286 if (I != Slab->Regions.begin()) {
287 auto J = std::prev(x: I);
288 ExecutorAddrRange PrevRange(J->first, J->second.Size);
289 if (PrevRange.overlaps(Other: R))
290 return MakeBadRegionError(PrevRange, true);
291 }
292 if (I != Slab->Regions.end()) {
293 ExecutorAddrRange NextRange(I->first, I->second.Size);
294 if (NextRange.overlaps(Other: R))
295 return MakeBadRegionError(NextRange, false);
296 }
297
298 auto &RInfo = Slab->Regions[R.Start];
299 RInfo.Size = R.size();
300 return RInfo;
301}
302
303Expected<SimpleExecutorMemoryManager::RegionInfo &>
304SimpleExecutorMemoryManager::getRegionInfo(SlabInfo &Slab, ExecutorAddr A,
305 StringRef Context) {
306 auto I = Slab.Regions.find(x: A);
307 if (I == Slab.Regions.end())
308 return make_error<StringError>(
309 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
310 " does not correspond to the start of any initialized region",
311 Args: inconvertibleErrorCode());
312
313 return I->second;
314}
315
316Expected<SimpleExecutorMemoryManager::RegionInfo &>
317SimpleExecutorMemoryManager::getRegionInfo(ExecutorAddr A, StringRef Context) {
318 auto Slab = getSlabInfo(A, Context);
319 if (!Slab)
320 return Slab.takeError();
321
322 return getRegionInfo(Slab&: *Slab, A, Context);
323}
324
325llvm::orc::shared::CWrapperFunctionBuffer
326SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
327 size_t ArgSize) {
328 return shared::WrapperFunction<rt::sps_ci::MemMgrReserve::SPSSig>::handle(
329 ArgData, ArgSize,
330 Handler: shared::makeMethodWrapperHandler(
331 Method: &SimpleExecutorMemoryManager::reserve))
332 .release();
333}
334
335llvm::orc::shared::CWrapperFunctionBuffer
336SimpleExecutorMemoryManager::initializeWrapper(const char *ArgData,
337 size_t ArgSize) {
338 return shared::WrapperFunction<rt::sps_ci::MemMgrInitialize::SPSSig>::handle(
339 ArgData, ArgSize,
340 Handler: shared::makeMethodWrapperHandler(
341 Method: &SimpleExecutorMemoryManager::initialize))
342 .release();
343}
344
345llvm::orc::shared::CWrapperFunctionBuffer
346SimpleExecutorMemoryManager::deinitializeWrapper(const char *ArgData,
347 size_t ArgSize) {
348 return shared::WrapperFunction<rt::sps_ci::MemMgrDeinitialize::SPSSig>::
349 handle(ArgData, ArgSize,
350 Handler: shared::makeMethodWrapperHandler(
351 Method: &SimpleExecutorMemoryManager::deinitialize))
352 .release();
353}
354
355llvm::orc::shared::CWrapperFunctionBuffer
356SimpleExecutorMemoryManager::releaseWrapper(const char *ArgData,
357 size_t ArgSize) {
358 return shared::WrapperFunction<rt::sps_ci::MemMgrRelease::SPSSig>::handle(
359 ArgData, ArgSize,
360 Handler: shared::makeMethodWrapperHandler(
361 Method: &SimpleExecutorMemoryManager::release))
362 .release();
363}
364
365} // namespace rt_bootstrap
366} // end namespace orc
367} // end namespace llvm
368