1//===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
10
11#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12#include "llvm/Support/FormatVariadic.h"
13
14#define DEBUG_TYPE "orc"
15
16namespace llvm {
17namespace orc {
18namespace rt_bootstrap {
19
20SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
21 assert(Allocations.empty() && "shutdown not called?");
22}
23
24Expected<ExecutorAddr> SimpleExecutorMemoryManager::allocate(uint64_t Size) {
25 std::error_code EC;
26 auto MB = sys::Memory::allocateMappedMemory(
27 NumBytes: Size, NearBlock: nullptr, Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
28 if (EC)
29 return errorCodeToError(EC);
30 std::lock_guard<std::mutex> Lock(M);
31 assert(!Allocations.count(MB.base()) && "Duplicate allocation addr");
32 Allocations[MB.base()].Size = Size;
33 return ExecutorAddr::fromPtr(Ptr: MB.base());
34}
35
36Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) {
37 ExecutorAddr Base(~0ULL);
38 std::vector<shared::WrapperFunctionCall> DeallocationActions;
39 size_t SuccessfulFinalizationActions = 0;
40
41 if (FR.Segments.empty()) {
42 // NOTE: Finalizing nothing is currently a no-op. Should it be an error?
43 if (FR.Actions.empty())
44 return Error::success();
45 else
46 return make_error<StringError>(Args: "Finalization actions attached to empty "
47 "finalization request",
48 Args: inconvertibleErrorCode());
49 }
50
51 for (auto &Seg : FR.Segments)
52 Base = std::min(a: Base, b: Seg.Addr);
53
54 for (auto &ActPair : FR.Actions)
55 if (ActPair.Dealloc)
56 DeallocationActions.push_back(x: ActPair.Dealloc);
57
58 // Get the Allocation for this finalization.
59 size_t AllocSize = 0;
60 {
61 std::lock_guard<std::mutex> Lock(M);
62 auto I = Allocations.find(Val: Base.toPtr<void *>());
63 if (I == Allocations.end())
64 return make_error<StringError>(Args: "Attempt to finalize unrecognized "
65 "allocation " +
66 formatv(Fmt: "{0:x}", Vals: Base.getValue()),
67 Args: inconvertibleErrorCode());
68 AllocSize = I->second.Size;
69 I->second.DeallocationActions = std::move(DeallocationActions);
70 }
71 ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize);
72
73 // Bail-out function: this will run deallocation actions corresponding to any
74 // completed finalization actions, then deallocate memory.
75 auto BailOut = [&](Error Err) {
76 std::pair<void *, Allocation> AllocToDestroy;
77
78 // Get allocation to destroy.
79 {
80 std::lock_guard<std::mutex> Lock(M);
81 auto I = Allocations.find(Val: Base.toPtr<void *>());
82
83 // Check for missing allocation (effective a double free).
84 if (I == Allocations.end())
85 return joinErrors(
86 E1: std::move(Err),
87 E2: make_error<StringError>(Args: "No allocation entry found "
88 "for " +
89 formatv(Fmt: "{0:x}", Vals: Base.getValue()),
90 Args: inconvertibleErrorCode()));
91 AllocToDestroy = std::move(*I);
92 Allocations.erase(I);
93 }
94
95 // Run deallocation actions for all completed finalization actions.
96 while (SuccessfulFinalizationActions)
97 Err =
98 joinErrors(E1: std::move(Err), E2: FR.Actions[--SuccessfulFinalizationActions]
99 .Dealloc.runWithSPSRetErrorMerged());
100
101 // Deallocate memory.
102 sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size);
103 if (auto EC = sys::Memory::releaseMappedMemory(Block&: MB))
104 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
105
106 return Err;
107 };
108
109 // Copy content and apply permissions.
110 for (auto &Seg : FR.Segments) {
111
112 // Check segment ranges.
113 if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
114 return BailOut(make_error<StringError>(
115 Args: formatv(Fmt: "Segment {0:x} content size ({1:x} bytes) "
116 "exceeds segment size ({2:x} bytes)",
117 Vals: Seg.Addr.getValue(), Vals: Seg.Content.size(), Vals&: Seg.Size),
118 Args: inconvertibleErrorCode()));
119 ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
120 if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd))
121 return BailOut(make_error<StringError>(
122 Args: formatv(Fmt: "Segment {0:x} -- {1:x} crosses boundary of "
123 "allocation {2:x} -- {3:x}",
124 Vals: Seg.Addr.getValue(), Vals: SegEnd.getValue(), Vals: Base.getValue(),
125 Vals: AllocEnd.getValue()),
126 Args: inconvertibleErrorCode()));
127
128 char *Mem = Seg.Addr.toPtr<char *>();
129 if (!Seg.Content.empty())
130 memcpy(dest: Mem, src: Seg.Content.data(), n: Seg.Content.size());
131 memset(s: Mem + Seg.Content.size(), c: 0, n: Seg.Size - Seg.Content.size());
132 assert(Seg.Size <= std::numeric_limits<size_t>::max());
133 if (auto EC = sys::Memory::protectMappedMemory(
134 Block: {Mem, static_cast<size_t>(Seg.Size)},
135 Flags: toSysMemoryProtectionFlags(MP: Seg.RAG.Prot)))
136 return BailOut(errorCodeToError(EC));
137 if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec)
138 sys::Memory::InvalidateInstructionCache(Addr: Mem, Len: Seg.Size);
139 }
140
141 // Run finalization actions.
142 for (auto &ActPair : FR.Actions) {
143 if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged())
144 return BailOut(std::move(Err));
145 ++SuccessfulFinalizationActions;
146 }
147
148 return Error::success();
149}
150
151Error SimpleExecutorMemoryManager::deallocate(
152 const std::vector<ExecutorAddr> &Bases) {
153 std::vector<std::pair<void *, Allocation>> AllocPairs;
154 AllocPairs.reserve(n: Bases.size());
155
156 // Get allocation to destroy.
157 Error Err = Error::success();
158 {
159 std::lock_guard<std::mutex> Lock(M);
160 for (auto &Base : Bases) {
161 auto I = Allocations.find(Val: Base.toPtr<void *>());
162
163 // Check for missing allocation (effective a double free).
164 if (I != Allocations.end()) {
165 AllocPairs.push_back(x: std::move(*I));
166 Allocations.erase(I);
167 } else
168 Err = joinErrors(
169 E1: std::move(Err),
170 E2: make_error<StringError>(Args: "No allocation entry found "
171 "for " +
172 formatv(Fmt: "{0:x}", Vals: Base.getValue()),
173 Args: inconvertibleErrorCode()));
174 }
175 }
176
177 while (!AllocPairs.empty()) {
178 auto &P = AllocPairs.back();
179 Err = joinErrors(E1: std::move(Err), E2: deallocateImpl(Base: P.first, A&: P.second));
180 AllocPairs.pop_back();
181 }
182
183 return Err;
184}
185
186Error SimpleExecutorMemoryManager::shutdown() {
187
188 AllocationsMap AM;
189 {
190 std::lock_guard<std::mutex> Lock(M);
191 AM = std::move(Allocations);
192 }
193
194 Error Err = Error::success();
195 for (auto &KV : AM)
196 Err = joinErrors(E1: std::move(Err), E2: deallocateImpl(Base: KV.first, A&: KV.second));
197 return Err;
198}
199
200void SimpleExecutorMemoryManager::addBootstrapSymbols(
201 StringMap<ExecutorAddr> &M) {
202 M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(Ptr: this);
203 M[rt::SimpleExecutorMemoryManagerReserveWrapperName] =
204 ExecutorAddr::fromPtr(Ptr: &reserveWrapper);
205 M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] =
206 ExecutorAddr::fromPtr(Ptr: &finalizeWrapper);
207 M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] =
208 ExecutorAddr::fromPtr(Ptr: &deallocateWrapper);
209}
210
211Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) {
212 Error Err = Error::success();
213
214 while (!A.DeallocationActions.empty()) {
215 Err = joinErrors(E1: std::move(Err),
216 E2: A.DeallocationActions.back().runWithSPSRetErrorMerged());
217 A.DeallocationActions.pop_back();
218 }
219
220 sys::MemoryBlock MB(Base, A.Size);
221 if (auto EC = sys::Memory::releaseMappedMemory(Block&: MB))
222 Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC));
223
224 return Err;
225}
226
227llvm::orc::shared::CWrapperFunctionResult
228SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
229 size_t ArgSize) {
230 return shared::WrapperFunction<
231 rt::SPSSimpleExecutorMemoryManagerReserveSignature>::
232 handle(ArgData, ArgSize,
233 Handler: shared::makeMethodWrapperHandler(
234 Method: &SimpleExecutorMemoryManager::allocate))
235 .release();
236}
237
238llvm::orc::shared::CWrapperFunctionResult
239SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData,
240 size_t ArgSize) {
241 return shared::WrapperFunction<
242 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>::
243 handle(ArgData, ArgSize,
244 Handler: shared::makeMethodWrapperHandler(
245 Method: &SimpleExecutorMemoryManager::finalize))
246 .release();
247}
248
249llvm::orc::shared::CWrapperFunctionResult
250SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData,
251 size_t ArgSize) {
252 return shared::WrapperFunction<
253 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>::
254 handle(ArgData, ArgSize,
255 Handler: shared::makeMethodWrapperHandler(
256 Method: &SimpleExecutorMemoryManager::deallocate))
257 .release();
258}
259
260} // namespace rt_bootstrap
261} // end namespace orc
262} // end namespace llvm
263