1//===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
10#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
11#include "llvm/Support/Alignment.h"
12#include "llvm/Support/FormatVariadic.h"
13
14#define DEBUG_TYPE "orc"
15
16using namespace llvm::orc::shared;
17
18namespace llvm {
19namespace orc {
20
21Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>>
22EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
23 ExecutorProcessControl &EPC) {
24 SymbolAddrs SAs;
25 if (auto Err = EPC.getBootstrapSymbols(
26 Pairs: {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName},
27 {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName},
28 {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName},
29 {SAs.Deallocate,
30 rt::SimpleExecutorMemoryManagerDeallocateWrapperName},
31 {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionAllocActionName},
32 {SAs.DeregisterEHFrame,
33 rt::DeregisterEHFrameSectionAllocActionName}}))
34 return std::move(Err);
35 return std::make_unique<EPCGenericRTDyldMemoryManager>(args&: EPC, args: std::move(SAs));
36}
37
38EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager(
39 ExecutorProcessControl &EPC, SymbolAddrs SAs)
40 : EPC(EPC), SAs(std::move(SAs)) {
41 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n");
42}
43
44EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() {
45 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n");
46 if (!ErrMsg.empty())
47 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n";
48
49 Error Err = Error::success();
50 if (auto Err2 = EPC.callSPSWrapper<
51 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
52 WrapperFnAddr: SAs.Reserve, WrapperCallArgs&: Err, WrapperCallArgs&: SAs.Instance, WrapperCallArgs&: FinalizedAllocs)) {
53 // FIXME: Report errors through EPC once that functionality is available.
54 logAllUnhandledErrors(E: std::move(Err2), OS&: errs(), ErrorBanner: "");
55 return;
56 }
57
58 if (Err)
59 logAllUnhandledErrors(E: std::move(Err), OS&: errs(), ErrorBanner: "");
60}
61
62uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection(
63 uintptr_t Size, unsigned Alignment, unsigned SectionID,
64 StringRef SectionName) {
65 std::lock_guard<std::mutex> Lock(M);
66 LLVM_DEBUG({
67 dbgs() << "Allocator " << (void *)this << " allocating code section "
68 << SectionName << ": size = " << formatv("{0:x}", Size)
69 << " bytes, alignment = " << Alignment << "\n";
70 });
71 auto &Seg = Unmapped.back().CodeAllocs;
72 Seg.emplace_back(args&: Size, args&: Alignment);
73 return reinterpret_cast<uint8_t *>(
74 alignAddr(Addr: Seg.back().Contents.get(), Alignment: Align(Alignment)));
75}
76
77uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection(
78 uintptr_t Size, unsigned Alignment, unsigned SectionID,
79 StringRef SectionName, bool IsReadOnly) {
80 std::lock_guard<std::mutex> Lock(M);
81 LLVM_DEBUG({
82 dbgs() << "Allocator " << (void *)this << " allocating "
83 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName
84 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment "
85 << Alignment << ")\n";
86 });
87
88 auto &Seg =
89 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs;
90
91 Seg.emplace_back(args&: Size, args&: Alignment);
92 return reinterpret_cast<uint8_t *>(
93 alignAddr(Addr: Seg.back().Contents.get(), Alignment: Align(Alignment)));
94}
95
96void EPCGenericRTDyldMemoryManager::reserveAllocationSpace(
97 uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize,
98 Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) {
99
100 {
101 std::lock_guard<std::mutex> Lock(M);
102 // If there's already an error then bail out.
103 if (!ErrMsg.empty())
104 return;
105
106 if (CodeAlign > EPC.getPageSize()) {
107 ErrMsg = "Invalid code alignment in reserveAllocationSpace";
108 return;
109 }
110 if (RODataAlign > EPC.getPageSize()) {
111 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace";
112 return;
113 }
114 if (RWDataAlign > EPC.getPageSize()) {
115 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace";
116 return;
117 }
118 }
119
120 uint64_t TotalSize = 0;
121 TotalSize += alignTo(Value: CodeSize, Align: EPC.getPageSize());
122 TotalSize += alignTo(Value: RODataSize, Align: EPC.getPageSize());
123 TotalSize += alignTo(Value: RWDataSize, Align: EPC.getPageSize());
124
125 LLVM_DEBUG({
126 dbgs() << "Allocator " << (void *)this << " reserving "
127 << formatv("{0:x}", TotalSize) << " bytes.\n";
128 });
129
130 Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr()));
131 if (auto Err = EPC.callSPSWrapper<
132 rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
133 WrapperFnAddr: SAs.Reserve, WrapperCallArgs&: TargetAllocAddr, WrapperCallArgs&: SAs.Instance, WrapperCallArgs&: TotalSize)) {
134 std::lock_guard<std::mutex> Lock(M);
135 ErrMsg = toString(E: std::move(Err));
136 return;
137 }
138 if (!TargetAllocAddr) {
139 std::lock_guard<std::mutex> Lock(M);
140 ErrMsg = toString(E: TargetAllocAddr.takeError());
141 return;
142 }
143
144 std::lock_guard<std::mutex> Lock(M);
145 Unmapped.push_back(x: SectionAllocGroup());
146 Unmapped.back().RemoteCode = {
147 *TargetAllocAddr, ExecutorAddrDiff(alignTo(Value: CodeSize, Align: EPC.getPageSize()))};
148 Unmapped.back().RemoteROData = {
149 Unmapped.back().RemoteCode.End,
150 ExecutorAddrDiff(alignTo(Value: RODataSize, Align: EPC.getPageSize()))};
151 Unmapped.back().RemoteRWData = {
152 Unmapped.back().RemoteROData.End,
153 ExecutorAddrDiff(alignTo(Value: RWDataSize, Align: EPC.getPageSize()))};
154}
155
156bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() {
157 return true;
158}
159
160void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr,
161 uint64_t LoadAddr,
162 size_t Size) {
163 LLVM_DEBUG({
164 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame "
165 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n";
166 });
167 std::lock_guard<std::mutex> Lock(M);
168 // Bail out early if there's already an error.
169 if (!ErrMsg.empty())
170 return;
171
172 ExecutorAddr LA(LoadAddr);
173 for (auto &SecAllocGroup : llvm::reverse(C&: Unfinalized)) {
174 if (SecAllocGroup.RemoteCode.contains(Addr: LA) ||
175 SecAllocGroup.RemoteROData.contains(Addr: LA) ||
176 SecAllocGroup.RemoteRWData.contains(Addr: LA)) {
177 SecAllocGroup.UnfinalizedEHFrames.push_back(x: {LA, Size});
178 return;
179 }
180 }
181 ErrMsg = "eh-frame does not lie inside unfinalized alloc";
182}
183
184void EPCGenericRTDyldMemoryManager::deregisterEHFrames() {
185 // This is a no-op for us: We've registered a deallocation action for it.
186}
187
188void EPCGenericRTDyldMemoryManager::notifyObjectLoaded(
189 RuntimeDyld &Dyld, const object::ObjectFile &Obj) {
190 std::lock_guard<std::mutex> Lock(M);
191 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n");
192 for (auto &ObjAllocs : Unmapped) {
193 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.CodeAllocs,
194 NextAddr: ObjAllocs.RemoteCode.Start);
195 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.RODataAllocs,
196 NextAddr: ObjAllocs.RemoteROData.Start);
197 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.RWDataAllocs,
198 NextAddr: ObjAllocs.RemoteRWData.Start);
199 Unfinalized.push_back(x: std::move(ObjAllocs));
200 }
201 Unmapped.clear();
202}
203
204bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) {
205 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n");
206
207 // If there's an error then bail out here.
208 std::vector<SectionAllocGroup> SecAllocGroups;
209 {
210 std::lock_guard<std::mutex> Lock(M);
211 if (ErrMsg && !this->ErrMsg.empty()) {
212 *ErrMsg = std::move(this->ErrMsg);
213 return true;
214 }
215 std::swap(x&: SecAllocGroups, y&: Unfinalized);
216 }
217
218 // Loop over unfinalized objects to make finalization requests.
219 for (auto &SecAllocGroup : SecAllocGroups) {
220
221 MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read,
222 MemProt::Read | MemProt::Write};
223
224 ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode,
225 &SecAllocGroup.RemoteROData,
226 &SecAllocGroup.RemoteRWData};
227
228 std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs,
229 &SecAllocGroup.RODataAllocs,
230 &SecAllocGroup.RWDataAllocs};
231
232 tpctypes::FinalizeRequest FR;
233 std::unique_ptr<char[]> AggregateContents[3];
234
235 for (unsigned I = 0; I != 3; ++I) {
236 FR.Segments.push_back(x: {});
237 auto &Seg = FR.Segments.back();
238 Seg.RAG = SegMemProts[I];
239 Seg.Addr = RemoteAddrs[I]->Start;
240 for (auto &SecAlloc : *SegSections[I]) {
241 Seg.Size = alignTo(Value: Seg.Size, Align: SecAlloc.Align);
242 Seg.Size += SecAlloc.Size;
243 }
244 AggregateContents[I] = std::make_unique<char[]>(num: Seg.Size);
245 size_t SecOffset = 0;
246 for (auto &SecAlloc : *SegSections[I]) {
247 SecOffset = alignTo(Value: SecOffset, Align: SecAlloc.Align);
248 memcpy(dest: &AggregateContents[I][SecOffset],
249 src: reinterpret_cast<const char *>(
250 alignAddr(Addr: SecAlloc.Contents.get(), Alignment: Align(SecAlloc.Align))),
251 n: SecAlloc.Size);
252 SecOffset += SecAlloc.Size;
253 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into
254 // the aggregated content?
255 }
256 Seg.Content = {AggregateContents[I].get(), SecOffset};
257 }
258
259 for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames)
260 FR.Actions.push_back(
261 x: {.Finalize: cantFail(
262 ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
263 FnAddr: SAs.RegisterEHFrame, Args: Frame)),
264 .Dealloc: cantFail(
265 ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
266 FnAddr: SAs.DeregisterEHFrame, Args: Frame))});
267
268 // We'll also need to make an extra allocation for the eh-frame wrapper call
269 // arguments.
270 Error FinalizeErr = Error::success();
271 if (auto Err = EPC.callSPSWrapper<
272 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>(
273 WrapperFnAddr: SAs.Finalize, WrapperCallArgs&: FinalizeErr, WrapperCallArgs&: SAs.Instance, WrapperCallArgs: std::move(FR))) {
274 std::lock_guard<std::mutex> Lock(M);
275 this->ErrMsg = toString(E: std::move(Err));
276 dbgs() << "Serialization error: " << this->ErrMsg << "\n";
277 if (ErrMsg)
278 *ErrMsg = this->ErrMsg;
279 return true;
280 }
281 if (FinalizeErr) {
282 std::lock_guard<std::mutex> Lock(M);
283 this->ErrMsg = toString(E: std::move(FinalizeErr));
284 dbgs() << "Finalization error: " << this->ErrMsg << "\n";
285 if (ErrMsg)
286 *ErrMsg = this->ErrMsg;
287 return true;
288 }
289 }
290
291 return false;
292}
293
294void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs(
295 RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs,
296 ExecutorAddr NextAddr) {
297 for (auto &Alloc : Allocs) {
298 NextAddr.setValue(alignTo(Value: NextAddr.getValue(), Align: Alloc.Align));
299 LLVM_DEBUG({
300 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> "
301 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n";
302 });
303 Dyld.mapSectionAddress(LocalAddress: reinterpret_cast<const void *>(alignAddr(
304 Addr: Alloc.Contents.get(), Alignment: Align(Alloc.Align))),
305 TargetAddress: NextAddr.getValue());
306 Alloc.RemoteAddr = NextAddr;
307 // Only advance NextAddr if it was non-null to begin with,
308 // otherwise leave it as null.
309 if (NextAddr)
310 NextAddr += ExecutorAddrDiff(Alloc.Size);
311 }
312}
313
314} // end namespace orc
315} // end namespace llvm
316