1//===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
10#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
11#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
12#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
13#include "llvm/ExecutionEngine/Orc/Shared/SPSCI/SimpleNativeMemoryMapSPSCI.h"
14#include "llvm/ExecutionEngine/Orc/SimpleMemoryMapSPS.h"
15#include "llvm/Support/Alignment.h"
16#include "llvm/Support/FormatVariadic.h"
17
18#define DEBUG_TYPE "orc"
19
20using namespace llvm::orc::shared;
21
22namespace llvm {
23namespace orc {
24
25Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>>
26EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
27 ExecutorProcessControl &EPC) {
28 SymbolAddrs SAs;
29 if (auto Err = lookupAndApply(
30 JD&: EPC.getExecutionSession().getBootstrapJITDylib(),
31 PrepareFns: {recordAddr(Name: rt::sps_ci::SimpleNativeMemoryMapInstanceName,
32 A: &SAs.MemMgr.Instance),
33 recordProxy<sps::MemMgrReserveProxySpec>(P: &SAs.MemMgr.Reserve),
34 recordProxy<sps::MemMgrInitializeProxySpec>(P: &SAs.MemMgr.Initialize),
35 recordProxy<sps::MemMgrReleaseProxySpec>(P: &SAs.MemMgr.Release),
36 recordAddr(Name: rt::RegisterEHFrameSectionAllocActionName,
37 A: &SAs.RegisterEHFrame),
38 recordAddr(Name: rt::DeregisterEHFrameSectionAllocActionName,
39 A: &SAs.DeregisterEHFrame)}))
40 return std::move(Err);
41 return std::make_unique<EPCGenericRTDyldMemoryManager>(args&: EPC, args: std::move(SAs));
42}
43
44EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager(
45 ExecutorProcessControl &EPC, SymbolAddrs SAs)
46 : EPC(EPC), SAs(std::move(SAs)) {
47 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n");
48}
49
50EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() {
51 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n");
52 if (!ErrMsg.empty())
53 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n";
54
55 // FIXME: Report errors through EPC once that functionality is available.
56 if (auto Err = SAs.MemMgr.Release(EPC.getExecutionSession(),
57 SAs.MemMgr.Instance, FinalizedAllocs))
58 logAllUnhandledErrors(E: std::move(Err), OS&: errs(), ErrorBanner: "");
59}
60
61uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection(
62 uintptr_t Size, unsigned Alignment, unsigned SectionID,
63 StringRef SectionName) {
64 std::lock_guard<std::mutex> Lock(M);
65 LLVM_DEBUG({
66 dbgs() << "Allocator " << (void *)this << " allocating code section "
67 << SectionName << ": size = " << formatv("{0:x}", Size)
68 << " bytes, alignment = " << Alignment << "\n";
69 });
70 auto &Seg = Unmapped.back().CodeAllocs;
71 Seg.emplace_back(args&: Size, args&: Alignment);
72 return reinterpret_cast<uint8_t *>(
73 alignAddr(Addr: Seg.back().Contents.get(), Alignment: Align(Alignment)));
74}
75
76uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection(
77 uintptr_t Size, unsigned Alignment, unsigned SectionID,
78 StringRef SectionName, bool IsReadOnly) {
79 std::lock_guard<std::mutex> Lock(M);
80 LLVM_DEBUG({
81 dbgs() << "Allocator " << (void *)this << " allocating "
82 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName
83 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment "
84 << Alignment << ")\n";
85 });
86
87 auto &Seg =
88 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs;
89
90 Seg.emplace_back(args&: Size, args&: Alignment);
91 return reinterpret_cast<uint8_t *>(
92 alignAddr(Addr: Seg.back().Contents.get(), Alignment: Align(Alignment)));
93}
94
95void EPCGenericRTDyldMemoryManager::reserveAllocationSpace(
96 uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize,
97 Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) {
98
99 {
100 std::lock_guard<std::mutex> Lock(M);
101 // If there's already an error then bail out.
102 if (!ErrMsg.empty())
103 return;
104
105 if (CodeAlign > EPC.getPageSize()) {
106 ErrMsg = "Invalid code alignment in reserveAllocationSpace";
107 return;
108 }
109 if (RODataAlign > EPC.getPageSize()) {
110 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace";
111 return;
112 }
113 if (RWDataAlign > EPC.getPageSize()) {
114 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace";
115 return;
116 }
117 }
118
119 uint64_t TotalSize = 0;
120 TotalSize += alignTo(Value: CodeSize, Align: EPC.getPageSize());
121 TotalSize += alignTo(Value: RODataSize, Align: EPC.getPageSize());
122 TotalSize += alignTo(Value: RWDataSize, Align: EPC.getPageSize());
123
124 LLVM_DEBUG({
125 dbgs() << "Allocator " << (void *)this << " reserving "
126 << formatv("{0:x}", TotalSize) << " bytes.\n";
127 });
128
129 Expected<ExecutorAddr> TargetAllocAddr = SAs.MemMgr.Reserve(
130 EPC.getExecutionSession(), SAs.MemMgr.Instance, TotalSize);
131 if (!TargetAllocAddr) {
132 std::lock_guard<std::mutex> Lock(M);
133 ErrMsg = toString(E: TargetAllocAddr.takeError());
134 return;
135 }
136
137 std::lock_guard<std::mutex> Lock(M);
138 Unmapped.push_back(x: SectionAllocGroup());
139 Unmapped.back().RemoteCode = {
140 *TargetAllocAddr, ExecutorAddrDiff(alignTo(Value: CodeSize, Align: EPC.getPageSize()))};
141 Unmapped.back().RemoteROData = {
142 Unmapped.back().RemoteCode.End,
143 ExecutorAddrDiff(alignTo(Value: RODataSize, Align: EPC.getPageSize()))};
144 Unmapped.back().RemoteRWData = {
145 Unmapped.back().RemoteROData.End,
146 ExecutorAddrDiff(alignTo(Value: RWDataSize, Align: EPC.getPageSize()))};
147}
148
149bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() {
150 return true;
151}
152
153void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr,
154 uint64_t LoadAddr,
155 size_t Size) {
156 LLVM_DEBUG({
157 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame "
158 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n";
159 });
160 std::lock_guard<std::mutex> Lock(M);
161 // Bail out early if there's already an error.
162 if (!ErrMsg.empty())
163 return;
164
165 ExecutorAddr LA(LoadAddr);
166 for (auto &SecAllocGroup : llvm::reverse(C&: Unfinalized)) {
167 if (SecAllocGroup.RemoteCode.contains(Addr: LA) ||
168 SecAllocGroup.RemoteROData.contains(Addr: LA) ||
169 SecAllocGroup.RemoteRWData.contains(Addr: LA)) {
170 SecAllocGroup.UnfinalizedEHFrames.push_back(x: {LA, Size});
171 return;
172 }
173 }
174 ErrMsg = "eh-frame does not lie inside unfinalized alloc";
175}
176
177void EPCGenericRTDyldMemoryManager::deregisterEHFrames() {
178 // This is a no-op for us: We've registered a deallocation action for it.
179}
180
181void EPCGenericRTDyldMemoryManager::notifyObjectLoaded(
182 RuntimeDyld &Dyld, const object::ObjectFile &Obj) {
183 std::lock_guard<std::mutex> Lock(M);
184 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n");
185 for (auto &ObjAllocs : Unmapped) {
186 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.CodeAllocs,
187 NextAddr: ObjAllocs.RemoteCode.Start);
188 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.RODataAllocs,
189 NextAddr: ObjAllocs.RemoteROData.Start);
190 mapAllocsToRemoteAddrs(Dyld, SecAllocs&: ObjAllocs.RWDataAllocs,
191 NextAddr: ObjAllocs.RemoteRWData.Start);
192 Unfinalized.push_back(x: std::move(ObjAllocs));
193 }
194 Unmapped.clear();
195}
196
197bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) {
198 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n");
199
200 // If there's an error then bail out here.
201 std::vector<SectionAllocGroup> SecAllocGroups;
202 {
203 std::lock_guard<std::mutex> Lock(M);
204 if (ErrMsg && !this->ErrMsg.empty()) {
205 *ErrMsg = std::move(this->ErrMsg);
206 return true;
207 }
208 std::swap(x&: SecAllocGroups, y&: Unfinalized);
209 }
210
211 // Loop over unfinalized objects to make finalization requests.
212 for (auto &SecAllocGroup : SecAllocGroups) {
213
214 MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read,
215 MemProt::Read | MemProt::Write};
216
217 ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode,
218 &SecAllocGroup.RemoteROData,
219 &SecAllocGroup.RemoteRWData};
220
221 std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs,
222 &SecAllocGroup.RODataAllocs,
223 &SecAllocGroup.RWDataAllocs};
224
225 tpctypes::FinalizeRequest FR;
226 std::unique_ptr<char[]> AggregateContents[3];
227
228 for (unsigned I = 0; I != 3; ++I) {
229 FR.Segments.push_back(x: {});
230 auto &Seg = FR.Segments.back();
231 Seg.RAG = SegMemProts[I];
232 Seg.Addr = RemoteAddrs[I]->Start;
233 for (auto &SecAlloc : *SegSections[I]) {
234 Seg.Size = alignTo(Value: Seg.Size, Align: SecAlloc.Align);
235 Seg.Size += SecAlloc.Size;
236 }
237 AggregateContents[I] = std::make_unique<char[]>(num: Seg.Size);
238 size_t SecOffset = 0;
239 for (auto &SecAlloc : *SegSections[I]) {
240 SecOffset = alignTo(Value: SecOffset, Align: SecAlloc.Align);
241 memcpy(dest: &AggregateContents[I][SecOffset],
242 src: reinterpret_cast<const char *>(
243 alignAddr(Addr: SecAlloc.Contents.get(), Alignment: Align(SecAlloc.Align))),
244 n: SecAlloc.Size);
245 SecOffset += SecAlloc.Size;
246 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into
247 // the aggregated content?
248 }
249 Seg.Content = {AggregateContents[I].get(), SecOffset};
250 }
251
252 for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames)
253 FR.Actions.push_back(
254 x: {.Finalize: cantFail(
255 ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
256 FnAddr: SAs.RegisterEHFrame, Args: Frame)),
257 .Dealloc: cantFail(
258 ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
259 FnAddr: SAs.DeregisterEHFrame, Args: Frame))});
260
261 // We'll also need to make an extra allocation for the eh-frame wrapper call
262 // arguments.
263 Expected<ExecutorAddr> InitializeKey = SAs.MemMgr.Initialize(
264 EPC.getExecutionSession(), SAs.MemMgr.Instance, std::move(FR));
265 if (!InitializeKey) {
266 std::lock_guard<std::mutex> Lock(M);
267 this->ErrMsg = toString(E: InitializeKey.takeError());
268 dbgs() << "Finalization error: " << this->ErrMsg << "\n";
269 if (ErrMsg)
270 *ErrMsg = this->ErrMsg;
271 return true;
272 }
273 }
274
275 return false;
276}
277
278void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs(
279 RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs,
280 ExecutorAddr NextAddr) {
281 for (auto &Alloc : Allocs) {
282 NextAddr.setValue(alignTo(Value: NextAddr.getValue(), Align: Alloc.Align));
283 LLVM_DEBUG({
284 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> "
285 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n";
286 });
287 Dyld.mapSectionAddress(LocalAddress: reinterpret_cast<const void *>(alignAddr(
288 Addr: Alloc.Contents.get(), Alignment: Align(Alloc.Align))),
289 TargetAddress: NextAddr.getValue());
290 Alloc.RemoteAddr = NextAddr;
291 // Only advance NextAddr if it was non-null to begin with,
292 // otherwise leave it as null.
293 if (NextAddr)
294 NextAddr += ExecutorAddrDiff(Alloc.Size);
295 }
296}
297
298} // end namespace orc
299} // end namespace llvm
300