1//=== MapperJITLinkMemoryManager.cpp - Memory management with MemoryMapper ===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
10
11#include "llvm/ExecutionEngine/JITLink/JITLink.h"
12#include "llvm/Support/Error.h"
13#include "llvm/Support/Process.h"
14
15using namespace llvm::jitlink;
16
17namespace llvm {
18namespace orc {
19
20class MapperJITLinkMemoryManager::InFlightAlloc
21 : public JITLinkMemoryManager::InFlightAlloc {
22public:
23 InFlightAlloc(MapperJITLinkMemoryManager &Parent, LinkGraph &G,
24 ExecutorAddr AllocAddr,
25 std::vector<MemoryMapper::AllocInfo::SegInfo> Segs)
26 : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {}
27
28 void finalize(OnFinalizedFunction OnFinalize) override {
29 MemoryMapper::AllocInfo AI;
30 AI.MappingBase = AllocAddr;
31
32 std::swap(x&: AI.Segments, y&: Segs);
33 std::swap(x&: AI.Actions, y&: G.allocActions());
34
35 Parent.Mapper->initialize(AI, OnInitialized: [OnFinalize = std::move(OnFinalize)](
36 Expected<ExecutorAddr> Result) mutable {
37 if (!Result) {
38 OnFinalize(Result.takeError());
39 return;
40 }
41
42 OnFinalize(FinalizedAlloc(*Result));
43 });
44 }
45
46 void abandon(OnAbandonedFunction OnFinalize) override {
47 Parent.Mapper->deinitialize(Allocations: {AllocAddr}, OnDeInitialized: std::move(OnFinalize));
48 }
49
50private:
51 MapperJITLinkMemoryManager &Parent;
52 LinkGraph &G;
53 ExecutorAddr AllocAddr;
54 std::vector<MemoryMapper::AllocInfo::SegInfo> Segs;
55};
56
57MapperJITLinkMemoryManager::MapperJITLinkMemoryManager(
58 size_t ReservationGranularity, std::unique_ptr<MemoryMapper> Mapper,
59 bool ColocatePerJITDylib)
60 : ReservationUnits(ReservationGranularity),
61 ColocatePerJITDylib(ColocatePerJITDylib), Mapper(std::move(Mapper)) {}
62
63// Lazily creates and stores Key's pool on first access. Pools are
64// heap-allocated behind unique_ptr so the returned reference survives map
65// rehashing, and AMAllocator (declared before the pools) outlives them. Also
66// registers for a destruction notification the first time a JITDylib is
67// seen, so its pool can be freed in notifyDestroying().
68MapperJITLinkMemoryManager::AvailableMemoryMap &
69MapperJITLinkMemoryManager::getAvailableMemoryLocked(const JITLinkDylib *Key) {
70 auto [It, Inserted] = Pools.try_emplace(Key);
71 if (Inserted && Key)
72 const_cast<JITLinkDylib *>(Key)->notifyOnDestruction(MemMgr&: *this);
73 auto &Pool = It->second.AvailPool;
74 if (!Pool)
75 Pool = std::make_unique<AvailableMemoryMap>(args&: AMAllocator);
76 return *Pool;
77}
78
79void MapperJITLinkMemoryManager::notifyDestroying(JITLinkDylib &JD) {
80 std::lock_guard<std::mutex> Lock(Mutex);
81 Pools.erase(Val: &JD);
82}
83
84void MapperJITLinkMemoryManager::denySlabGrowth() {
85 std::lock_guard<std::mutex> Lock(Mutex);
86 OnSlabGrow = [](const JITLinkDylib *JD, size_t RequestedSize) -> Error {
87 return createStringError(
88 Fmt: "JITDylib '%s' slab exhausted (single-slab mode); enable slab "
89 "growth to allow further reservations (%zu bytes requested)",
90 Vals: JD ? JD->getName().c_str() : "<unnamed>", Vals: RequestedSize);
91 };
92}
93
94void MapperJITLinkMemoryManager::allowSlabGrowth() {
95 std::lock_guard<std::mutex> Lock(Mutex);
96 OnSlabGrow = [](const JITLinkDylib *, size_t) -> Error {
97 return Error::success();
98 };
99}
100
101void MapperJITLinkMemoryManager::setSlabPolicy(SlabGrowthPolicy Policy) {
102 std::lock_guard<std::mutex> Lock(Mutex);
103 OnSlabGrow = std::move(Policy);
104}
105
106Expected<std::unique_ptr<MapperJITLinkMemoryManager>>
107createColocatingInProcessMemoryManager(
108 std::optional<size_t> ReservationGranularity) {
109 auto Mapper = InProcessMemoryMapper::Create();
110 if (!Mapper)
111 return Mapper.takeError();
112 return std::make_unique<MapperJITLinkMemoryManager>(
113 args: ReservationGranularity.value_or(
114 u: MapperJITLinkMemoryManager::defaultSlabSize()),
115 args: std::move(*Mapper),
116 /*ColocatePerJITDylib=*/args: true);
117}
118
119void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G,
120 OnAllocatedFunction OnAllocated) {
121 BasicLayout BL(G);
122
123 // find required address space
124 auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize: Mapper->getPageSize());
125 if (!SegsSizes) {
126 OnAllocated(SegsSizes.takeError());
127 return;
128 }
129
130 auto TotalSize = SegsSizes->total();
131
132 // Which pool to draw from: the JITDylib's own pool when colocating
133 // per-JITDylib, otherwise a single shared (nullptr) pool.
134 const JITLinkDylib *PoolKey = ColocatePerJITDylib ? JD : nullptr;
135
136 auto CompleteAllocation = [this, PoolKey, &G, BL = std::move(BL),
137 OnAllocated = std::move(OnAllocated)](
138 Expected<ExecutorAddrRange> Result) mutable {
139 if (!Result) {
140 Mutex.unlock();
141 return OnAllocated(Result.takeError());
142 }
143
144 auto NextSegAddr = Result->Start;
145
146 std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos;
147
148 for (auto &KV : BL.segments()) {
149 auto &AG = KV.first;
150 auto &Seg = KV.second;
151
152 auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize;
153
154 Seg.Addr = NextSegAddr;
155 Seg.WorkingMem = Mapper->prepare(G, Addr: NextSegAddr, ContentSize: TotalSize);
156
157 NextSegAddr += alignTo(Value: TotalSize, Align: Mapper->getPageSize());
158
159 MemoryMapper::AllocInfo::SegInfo SI;
160 SI.Offset = Seg.Addr - Result->Start;
161 SI.ContentSize = Seg.ContentSize;
162 SI.ZeroFillSize = Seg.ZeroFillSize;
163 SI.AG = AG;
164 SI.WorkingMem = Seg.WorkingMem;
165
166 SegInfos.push_back(x: SI);
167 }
168
169 UsedMemory.insert(KV: {Result->Start, NextSegAddr - Result->Start});
170 AllocPoolKey[Result->Start] = PoolKey;
171
172 if (NextSegAddr < Result->End) {
173 // Save the remaining memory for reuse by later allocations from the same
174 // pool.
175 getAvailableMemoryLocked(Key: PoolKey).insert(a: NextSegAddr, b: Result->End - 1,
176 y: true);
177 }
178 Mutex.unlock();
179
180 if (auto Err = BL.apply()) {
181 OnAllocated(std::move(Err));
182 return;
183 }
184
185 OnAllocated(std::make_unique<InFlightAlloc>(args&: *this, args&: G, args&: Result->Start,
186 args: std::move(SegInfos)));
187 };
188
189 Mutex.lock();
190
191 // find an already reserved range in this pool that is large enough
192 ExecutorAddrRange SelectedRange{};
193
194 AvailableMemoryMap &Avail = getAvailableMemoryLocked(Key: PoolKey);
195 for (AvailableMemoryMap::iterator It = Avail.begin(); It != Avail.end();
196 It++) {
197 if (It.stop() - It.start() + 1 >= TotalSize) {
198 SelectedRange = ExecutorAddrRange(It.start(), It.stop() + 1);
199 It.erase();
200 break;
201 }
202 }
203
204 if (SelectedRange.empty()) { // no already reserved range was found
205 // A reservation is needed. If this pool already owns one, reserving
206 // another (possibly out-of-range) slab may place its objects out of range,
207 // so consult the growth policy first.
208 bool IsFirstReservation = !Pools[PoolKey].Reserved;
209 Pools[PoolKey].Reserved = true;
210 auto TotalAllocation = alignTo(Value: TotalSize, Align: ReservationUnits);
211 if (!IsFirstReservation) {
212 if (auto Err = OnSlabGrow(JD, TotalAllocation))
213 return CompleteAllocation(std::move(Err));
214 }
215 Mapper->reserve(NumBytes: TotalAllocation,
216 OnReserved: [this, PoolKey, IsFirstReservation,
217 CompleteAllocation = std::move(CompleteAllocation)](
218 Expected<ExecutorAddrRange> Result) mutable {
219 // If this was the pool's first reservation attempt and it
220 // failed, undo the Reserved flag so a later attempt is
221 // treated as the first one again, rather than
222 // incorrectly triggering the growth policy for a pool
223 // that never got a slab.
224 if (!Result && IsFirstReservation)
225 Pools[PoolKey].Reserved = false;
226 CompleteAllocation(std::move(Result));
227 });
228 } else {
229 CompleteAllocation(SelectedRange);
230 }
231}
232
233void MapperJITLinkMemoryManager::deallocate(
234 std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {
235 std::vector<ExecutorAddr> Bases;
236 Bases.reserve(n: Allocs.size());
237 for (auto &FA : Allocs) {
238 ExecutorAddr Addr = FA.getAddress();
239 Bases.push_back(x: Addr);
240 }
241
242 Mapper->deinitialize(Allocations: Bases, OnDeInitialized: [this, Allocs = std::move(Allocs),
243 OnDeallocated = std::move(OnDeallocated)](
244 llvm::Error Err) mutable {
245 // TODO: How should we treat memory that we fail to deinitialize?
246 // We're currently bailing out and treating it as "burned" -- should we
247 // require that a failure to deinitialize still reset the memory so that
248 // we can reclaim it?
249 if (Err) {
250 for (auto &FA : Allocs)
251 FA.release();
252 OnDeallocated(std::move(Err));
253 return;
254 }
255
256 {
257 std::lock_guard<std::mutex> Lock(Mutex);
258
259 for (auto &FA : Allocs) {
260 ExecutorAddr Addr = FA.getAddress();
261 ExecutorAddrDiff Size = UsedMemory[Addr];
262
263 UsedMemory.erase(Val: Addr);
264
265 // Return the range to the pool it was allocated from.
266 const JITLinkDylib *PoolKey = AllocPoolKey.lookup(Val: Addr);
267 AllocPoolKey.erase(Val: Addr);
268 getAvailableMemoryLocked(Key: PoolKey).insert(a: Addr, b: Addr + Size - 1, y: true);
269
270 FA.release();
271 }
272 }
273
274 OnDeallocated(Error::success());
275 });
276}
277
278} // end namespace orc
279} // end namespace llvm
280