1//===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
10#include "llvm/ExecutionEngine/Orc/Shared/Mangler.h"
11#include "llvm/TargetParser/Host.h"
12#include "llvm/TargetParser/Triple.h"
13
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ExecutionEngine/Orc/Shared/SPSCI/SimpleNativeMemoryMapSPSCI.h"
16#include "llvm/Support/FormatVariadic.h"
17
18#define DEBUG_TYPE "orc"
19
20namespace llvm {
21namespace orc {
22namespace rt_bootstrap {
23
24SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
25 assert(Slabs.empty() && "shutdown not called?");
26}
27
28Expected<ExecutorAddr> SimpleExecutorMemoryManager::reserve(uint64_t Size) {
29 std::error_code EC;
30 auto MB = sys::Memory::allocateMappedMemory(
31 NumBytes: Size, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
32 if (EC)
33 return errorCodeToError(EC);
34 std::lock_guard<std::mutex> Lock(M);
35 assert(!Slabs.count(MB.base()) && "Duplicate allocation addr");
36 Slabs[MB.base()].Size = Size;
37 return ExecutorAddr::fromPtr(Ptr: MB.base());
38}
39
40Expected<ExecutorAddr>
41SimpleExecutorMemoryManager::initialize(tpctypes::FinalizeRequest &FR) {
42 if (FR.Segments.empty()) {
43 if (FR.Actions.empty())
44 return make_error<StringError>(Args: "Finalization request is empty",
45 Args: inconvertibleErrorCode());
46 else
47 return make_error<StringError>(Args: "Finalization actions attached to empty "
48 "finalization request",
49 Args: inconvertibleErrorCode());
50 }
51
52 ExecutorAddrRange RR(FR.Segments.front().Addr, FR.Segments.front().Addr);
53
54 std::vector<sys::MemoryBlock> MBsToReset;
55 llvm::scope_exit ResetMBs([&]() {
56 for (auto &MB : MBsToReset)
57 sys::Memory::protectMappedMemory(Block: MB, Flags: sys::Memory::MF_READ |
58 sys::Memory::MF_WRITE);
59 sys::Memory::InvalidateInstructionCache(Addr: RR.Start.toPtr<void *>(),
60 Len: RR.size());
61 });
62
63 // Copy content and apply permissions.
64 for (auto &Seg : FR.Segments) {
65 RR.Start = std::min(a: RR.Start, b: Seg.Addr);
66 RR.End = std::max(a: RR.End, b: Seg.Addr + Seg.Size);
67
68 // Check segment ranges.
69 if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
70 return make_error<StringError>(
71 Args: formatv(Fmt: "Segment {0:x} content size ({1:x} bytes) "
72 "exceeds segment size ({2:x} bytes)",
73 Vals: Seg.Addr.getValue(), Vals: Seg.Content.size(), Vals&: Seg.Size),
74 Args: inconvertibleErrorCode());
75 ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
76 if (LLVM_UNLIKELY(Seg.Addr < RR.Start || SegEnd > RR.End))
77 return make_error<StringError>(
78 Args: formatv(Fmt: "Segment {0:x} -- {1:x} crosses boundary of "
79 "allocation {2:x} -- {3:x}",
80 Vals&: Seg.Addr, Vals&: SegEnd, Vals&: RR.Start, Vals&: RR.End),
81 Args: inconvertibleErrorCode());
82
83 char *Mem = Seg.Addr.toPtr<char *>();
84 if (!Seg.Content.empty())
85 memcpy(dest: Mem, src: Seg.Content.data(), n: Seg.Content.size());
86 memset(s: Mem + Seg.Content.size(), c: 0, n: Seg.Size - Seg.Content.size());
87 assert(Seg.Size <= std::numeric_limits<size_t>::max());
88
89 sys::MemoryBlock MB(Mem, Seg.Size);
90 if (auto EC = sys::Memory::protectMappedMemory(
91 Block: MB, Flags: toSysMemoryProtectionFlags(MP: Seg.RAG.Prot)))
92 return errorCodeToError(EC);
93
94 MBsToReset.push_back(x: MB);
95
96 if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec)
97 sys::Memory::InvalidateInstructionCache(Addr: Mem, Len: Seg.Size);
98 }
99
100 auto DeallocActions = runFinalizeActions(AAs&: FR.Actions);
101 if (!DeallocActions)
102 return DeallocActions.takeError();
103
104 {
105 std::lock_guard<std::mutex> Lock(M);
106 auto Region = createRegionInfo(R: RR, Context: "In initialize");
107 if (!Region)
108 return Region.takeError();
109 Region->DeallocActions = std::move(*DeallocActions);
110 }
111
112 // Successful initialization.
113 ResetMBs.release();
114
115 return RR.Start;
116}
117
118Error SimpleExecutorMemoryManager::deinitialize(
119 const std::vector<ExecutorAddr> &InitKeys) {
120 Error Err = Error::success();
121
122 for (auto &KeyAddr : llvm::reverse(C: InitKeys)) {
123 std::vector<shared::WrapperFunctionCall> DeallocActions;
124 {
125 std::scoped_lock<std::mutex> Lock(M);
126 auto Slab = getSlabInfo(A: KeyAddr, Context: "In deinitialize");
127 if (!Slab) {
128 Err = joinErrors(E1: std::move(Err), E2: Slab.takeError());
129 continue;
130 }
131
132 auto RI = getRegionInfo(Slab&: *Slab, A: KeyAddr, Context: "In deinitialize");
133 if (!RI) {
134 Err = joinErrors(E1: std::move(Err), E2: RI.takeError());
135 continue;
136 }
137
138 DeallocActions = std::move(RI->DeallocActions);
139 }
140
141 Err = joinErrors(E1: std::move(Err),
142 E2: runDeallocActions(DAs: std::move(DeallocActions)));
143 }
144
145 return Err;
146}
147
148Error SimpleExecutorMemoryManager::release(
149 const std::vector<ExecutorAddr> &Bases) {
150 Error Err = Error::success();
151
152 // TODO: Prohibit new initializations within the slabs being removed?
153 for (auto &Base : llvm::reverse(C: Bases)) {
154 std::vector<shared::WrapperFunctionCall> DeallocActions;
155 sys::MemoryBlock MB;
156
157 {
158 std::scoped_lock<std::mutex> Lock(M);
159
160 auto SlabI = Slabs.find(x: Base.toPtr<void *>());
161 if (SlabI == Slabs.end()) {
162 Err = joinErrors(
163 E1: std::move(Err),
164 E2: make_error<StringError>(Args: "In release, " + formatv(Fmt: "{0:x}", Vals: Base) +
165 " is not part of any reserved "
166 "address range",
167 Args: inconvertibleErrorCode()));
168 continue;
169 }
170
171 auto &Slab = SlabI->second;
172
173 for (auto &[Addr, Region] : Slab.Regions)
174 llvm::copy(Range&: Region.DeallocActions, Out: back_inserter(x&: DeallocActions));
175
176 MB = {Base.toPtr<void *>(), Slab.Size};
177
178 Slabs.erase(position: SlabI);
179 }
180
181 Err = joinErrors(E1: std::move(Err), E2: runDeallocActions(DAs: DeallocActions));
182 if (auto EC = sys::Memory::releaseMappedMemory(Block&: MB))
183 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
184 }
185
186 return Err;
187}
188
189Error SimpleExecutorMemoryManager::shutdown() {
190
191 // TODO: Prevent new allocations during shutdown.
192 std::vector<ExecutorAddr> Bases;
193 {
194 std::scoped_lock<std::mutex> Lock(M);
195 for (auto &[Base, Slab] : Slabs)
196 Bases.push_back(x: ExecutorAddr::fromPtr(Ptr: Base));
197 }
198
199 return release(Bases);
200}
201
202void SimpleExecutorMemoryManager::addBootstrapSymbols(
203 StringMap<ExecutorAddr> &M) {
204 Mangler Mangle{Triple(sys::getProcessTriple())};
205 namespace sps_ci = rt::sps_ci;
206 M[Mangle.mangledCopy(Name: sps_ci::SimpleNativeMemoryMapInstanceName)] =
207 ExecutorAddr::fromPtr(Ptr: this);
208 M[Mangle.mangledCopy(Name: sps_ci::MemMgrReserve::Name)] =
209 ExecutorAddr::fromPtr(Ptr: reserveWrapper);
210 M[Mangle.mangledCopy(Name: sps_ci::MemMgrInitialize::Name)] =
211 ExecutorAddr::fromPtr(Ptr: initializeWrapper);
212 M[Mangle.mangledCopy(Name: sps_ci::MemMgrDeinitialize::Name)] =
213 ExecutorAddr::fromPtr(Ptr: deinitializeWrapper);
214 M[Mangle.mangledCopy(Name: sps_ci::MemMgrRelease::Name)] =
215 ExecutorAddr::fromPtr(Ptr: releaseWrapper);
216}
217
218Expected<SimpleExecutorMemoryManager::SlabInfo &>
219SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddr A, StringRef Context) {
220 auto MakeBadSlabError = [&]() {
221 return make_error<StringError>(
222 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
223 " is not part of any reserved address range",
224 Args: inconvertibleErrorCode());
225 };
226
227 auto I = Slabs.upper_bound(x: A.toPtr<void *>());
228 if (I == Slabs.begin())
229 return MakeBadSlabError();
230 --I;
231 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
232 .contains(Addr: A))
233 return MakeBadSlabError();
234
235 return I->second;
236}
237
238Expected<SimpleExecutorMemoryManager::SlabInfo &>
239SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddrRange R,
240 StringRef Context) {
241 auto MakeBadSlabError = [&]() {
242 return make_error<StringError>(
243 Args: Context + ", range " + formatv(Fmt: "{0:x}", Vals&: R) +
244 " is not part of any reserved address range",
245 Args: inconvertibleErrorCode());
246 };
247
248 auto I = Slabs.upper_bound(x: R.Start.toPtr<void *>());
249 if (I == Slabs.begin())
250 return MakeBadSlabError();
251 --I;
252 if (!ExecutorAddrRange(ExecutorAddr::fromPtr(Ptr: I->first), I->second.Size)
253 .contains(Other: R))
254 return MakeBadSlabError();
255
256 return I->second;
257}
258
259Expected<SimpleExecutorMemoryManager::RegionInfo &>
260SimpleExecutorMemoryManager::createRegionInfo(ExecutorAddrRange R,
261 StringRef Context) {
262
263 auto Slab = getSlabInfo(R, Context);
264 if (!Slab)
265 return Slab.takeError();
266
267 auto MakeBadRegionError = [&](ExecutorAddrRange Other, bool Prev) {
268 return make_error<StringError>(Args: Context + ", region " + formatv(Fmt: "{0:x}", Vals&: R) +
269 " overlaps " +
270 (Prev ? "previous" : "following") +
271 " region " + formatv(Fmt: "{0:x}", Vals&: Other),
272 Args: inconvertibleErrorCode());
273 };
274
275 auto I = Slab->Regions.upper_bound(x: R.Start);
276 if (I != Slab->Regions.begin()) {
277 auto J = std::prev(x: I);
278 ExecutorAddrRange PrevRange(J->first, J->second.Size);
279 if (PrevRange.overlaps(Other: R))
280 return MakeBadRegionError(PrevRange, true);
281 }
282 if (I != Slab->Regions.end()) {
283 ExecutorAddrRange NextRange(I->first, I->second.Size);
284 if (NextRange.overlaps(Other: R))
285 return MakeBadRegionError(NextRange, false);
286 }
287
288 auto &RInfo = Slab->Regions[R.Start];
289 RInfo.Size = R.size();
290 return RInfo;
291}
292
293Expected<SimpleExecutorMemoryManager::RegionInfo &>
294SimpleExecutorMemoryManager::getRegionInfo(SlabInfo &Slab, ExecutorAddr A,
295 StringRef Context) {
296 auto I = Slab.Regions.find(x: A);
297 if (I == Slab.Regions.end())
298 return make_error<StringError>(
299 Args: Context + ", address " + formatv(Fmt: "{0:x}", Vals&: A) +
300 " does not correspond to the start of any initialized region",
301 Args: inconvertibleErrorCode());
302
303 return I->second;
304}
305
306Expected<SimpleExecutorMemoryManager::RegionInfo &>
307SimpleExecutorMemoryManager::getRegionInfo(ExecutorAddr A, StringRef Context) {
308 auto Slab = getSlabInfo(A, Context);
309 if (!Slab)
310 return Slab.takeError();
311
312 return getRegionInfo(Slab&: *Slab, A, Context);
313}
314
315llvm::orc::shared::CWrapperFunctionBuffer
316SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
317 size_t ArgSize) {
318 return shared::WrapperFunction<rt::sps_ci::MemMgrReserve::SPSSig>::handle(
319 ArgData, ArgSize,
320 Handler: shared::makeMethodWrapperHandler(
321 Method: &SimpleExecutorMemoryManager::reserve))
322 .release();
323}
324
325llvm::orc::shared::CWrapperFunctionBuffer
326SimpleExecutorMemoryManager::initializeWrapper(const char *ArgData,
327 size_t ArgSize) {
328 return shared::WrapperFunction<rt::sps_ci::MemMgrInitialize::SPSSig>::handle(
329 ArgData, ArgSize,
330 Handler: shared::makeMethodWrapperHandler(
331 Method: &SimpleExecutorMemoryManager::initialize))
332 .release();
333}
334
335llvm::orc::shared::CWrapperFunctionBuffer
336SimpleExecutorMemoryManager::deinitializeWrapper(const char *ArgData,
337 size_t ArgSize) {
338 return shared::WrapperFunction<rt::sps_ci::MemMgrDeinitialize::SPSSig>::
339 handle(ArgData, ArgSize,
340 Handler: shared::makeMethodWrapperHandler(
341 Method: &SimpleExecutorMemoryManager::deinitialize))
342 .release();
343}
344
345llvm::orc::shared::CWrapperFunctionBuffer
346SimpleExecutorMemoryManager::releaseWrapper(const char *ArgData,
347 size_t ArgSize) {
348 return shared::WrapperFunction<rt::sps_ci::MemMgrRelease::SPSSig>::handle(
349 ArgData, ArgSize,
350 Handler: shared::makeMethodWrapperHandler(
351 Method: &SimpleExecutorMemoryManager::release))
352 .release();
353}
354
355} // namespace rt_bootstrap
356} // end namespace orc
357} // end namespace llvm
358