| 1 | //=== MapperJITLinkMemoryManager.cpp - Memory management with MemoryMapper ===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" |
| 10 | |
| 11 | #include "llvm/ExecutionEngine/JITLink/JITLink.h" |
| 12 | #include "llvm/Support/Error.h" |
| 13 | #include "llvm/Support/Process.h" |
| 14 | |
| 15 | using namespace llvm::jitlink; |
| 16 | |
| 17 | namespace llvm { |
| 18 | namespace orc { |
| 19 | |
| 20 | class MapperJITLinkMemoryManager::InFlightAlloc |
| 21 | : public JITLinkMemoryManager::InFlightAlloc { |
| 22 | public: |
| 23 | InFlightAlloc(MapperJITLinkMemoryManager &Parent, LinkGraph &G, |
| 24 | ExecutorAddr AllocAddr, |
| 25 | std::vector<MemoryMapper::AllocInfo::SegInfo> Segs) |
| 26 | : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {} |
| 27 | |
| 28 | void finalize(OnFinalizedFunction OnFinalize) override { |
| 29 | MemoryMapper::AllocInfo AI; |
| 30 | AI.MappingBase = AllocAddr; |
| 31 | |
| 32 | std::swap(x&: AI.Segments, y&: Segs); |
| 33 | std::swap(x&: AI.Actions, y&: G.allocActions()); |
| 34 | |
| 35 | Parent.Mapper->initialize(AI, OnInitialized: [OnFinalize = std::move(OnFinalize)]( |
| 36 | Expected<ExecutorAddr> Result) mutable { |
| 37 | if (!Result) { |
| 38 | OnFinalize(Result.takeError()); |
| 39 | return; |
| 40 | } |
| 41 | |
| 42 | OnFinalize(FinalizedAlloc(*Result)); |
| 43 | }); |
| 44 | } |
| 45 | |
| 46 | void abandon(OnAbandonedFunction OnFinalize) override { |
| 47 | Parent.Mapper->deinitialize(Allocations: {AllocAddr}, OnDeInitialized: std::move(OnFinalize)); |
| 48 | } |
| 49 | |
| 50 | private: |
| 51 | MapperJITLinkMemoryManager &Parent; |
| 52 | LinkGraph &G; |
| 53 | ExecutorAddr AllocAddr; |
| 54 | std::vector<MemoryMapper::AllocInfo::SegInfo> Segs; |
| 55 | }; |
| 56 | |
| 57 | MapperJITLinkMemoryManager::MapperJITLinkMemoryManager( |
| 58 | size_t ReservationGranularity, std::unique_ptr<MemoryMapper> Mapper, |
| 59 | bool ColocatePerJITDylib) |
| 60 | : ReservationUnits(ReservationGranularity), |
| 61 | ColocatePerJITDylib(ColocatePerJITDylib), Mapper(std::move(Mapper)) {} |
| 62 | |
| 63 | // Lazily creates and stores Key's pool on first access. Pools are |
| 64 | // heap-allocated behind unique_ptr so the returned reference survives map |
| 65 | // rehashing, and AMAllocator (declared before the pools) outlives them. Also |
| 66 | // registers for a destruction notification the first time a JITDylib is |
| 67 | // seen, so its pool can be freed in notifyDestroying(). |
| 68 | MapperJITLinkMemoryManager::AvailableMemoryMap & |
| 69 | MapperJITLinkMemoryManager::getAvailableMemoryLocked(const JITLinkDylib *Key) { |
| 70 | auto [It, Inserted] = Pools.try_emplace(Key); |
| 71 | if (Inserted && Key) |
| 72 | const_cast<JITLinkDylib *>(Key)->notifyOnDestruction(MemMgr&: *this); |
| 73 | auto &Pool = It->second.AvailPool; |
| 74 | if (!Pool) |
| 75 | Pool = std::make_unique<AvailableMemoryMap>(args&: AMAllocator); |
| 76 | return *Pool; |
| 77 | } |
| 78 | |
| 79 | void MapperJITLinkMemoryManager::notifyDestroying(JITLinkDylib &JD) { |
| 80 | std::lock_guard<std::mutex> Lock(Mutex); |
| 81 | Pools.erase(Val: &JD); |
| 82 | } |
| 83 | |
| 84 | void MapperJITLinkMemoryManager::denySlabGrowth() { |
| 85 | std::lock_guard<std::mutex> Lock(Mutex); |
| 86 | OnSlabGrow = [](const JITLinkDylib *JD, size_t RequestedSize) -> Error { |
| 87 | return createStringError( |
| 88 | Fmt: "JITDylib '%s' slab exhausted (single-slab mode); enable slab " |
| 89 | "growth to allow further reservations (%zu bytes requested)" , |
| 90 | Vals: JD ? JD->getName().c_str() : "<unnamed>" , Vals: RequestedSize); |
| 91 | }; |
| 92 | } |
| 93 | |
| 94 | void MapperJITLinkMemoryManager::allowSlabGrowth() { |
| 95 | std::lock_guard<std::mutex> Lock(Mutex); |
| 96 | OnSlabGrow = [](const JITLinkDylib *, size_t) -> Error { |
| 97 | return Error::success(); |
| 98 | }; |
| 99 | } |
| 100 | |
| 101 | void MapperJITLinkMemoryManager::setSlabPolicy(SlabGrowthPolicy Policy) { |
| 102 | std::lock_guard<std::mutex> Lock(Mutex); |
| 103 | OnSlabGrow = std::move(Policy); |
| 104 | } |
| 105 | |
| 106 | Expected<std::unique_ptr<MapperJITLinkMemoryManager>> |
| 107 | createColocatingInProcessMemoryManager( |
| 108 | std::optional<size_t> ReservationGranularity) { |
| 109 | auto Mapper = InProcessMemoryMapper::Create(); |
| 110 | if (!Mapper) |
| 111 | return Mapper.takeError(); |
| 112 | return std::make_unique<MapperJITLinkMemoryManager>( |
| 113 | args: ReservationGranularity.value_or( |
| 114 | u: MapperJITLinkMemoryManager::defaultSlabSize()), |
| 115 | args: std::move(*Mapper), |
| 116 | /*ColocatePerJITDylib=*/args: true); |
| 117 | } |
| 118 | |
| 119 | void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, |
| 120 | OnAllocatedFunction OnAllocated) { |
| 121 | BasicLayout BL(G); |
| 122 | |
| 123 | // find required address space |
| 124 | auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize: Mapper->getPageSize()); |
| 125 | if (!SegsSizes) { |
| 126 | OnAllocated(SegsSizes.takeError()); |
| 127 | return; |
| 128 | } |
| 129 | |
| 130 | auto TotalSize = SegsSizes->total(); |
| 131 | |
| 132 | // Which pool to draw from: the JITDylib's own pool when colocating |
| 133 | // per-JITDylib, otherwise a single shared (nullptr) pool. |
| 134 | const JITLinkDylib *PoolKey = ColocatePerJITDylib ? JD : nullptr; |
| 135 | |
| 136 | auto CompleteAllocation = [this, PoolKey, &G, BL = std::move(BL), |
| 137 | OnAllocated = std::move(OnAllocated)]( |
| 138 | Expected<ExecutorAddrRange> Result) mutable { |
| 139 | if (!Result) { |
| 140 | Mutex.unlock(); |
| 141 | return OnAllocated(Result.takeError()); |
| 142 | } |
| 143 | |
| 144 | auto NextSegAddr = Result->Start; |
| 145 | |
| 146 | std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos; |
| 147 | |
| 148 | for (auto &KV : BL.segments()) { |
| 149 | auto &AG = KV.first; |
| 150 | auto &Seg = KV.second; |
| 151 | |
| 152 | auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize; |
| 153 | |
| 154 | Seg.Addr = NextSegAddr; |
| 155 | Seg.WorkingMem = Mapper->prepare(G, Addr: NextSegAddr, ContentSize: TotalSize); |
| 156 | |
| 157 | NextSegAddr += alignTo(Value: TotalSize, Align: Mapper->getPageSize()); |
| 158 | |
| 159 | MemoryMapper::AllocInfo::SegInfo SI; |
| 160 | SI.Offset = Seg.Addr - Result->Start; |
| 161 | SI.ContentSize = Seg.ContentSize; |
| 162 | SI.ZeroFillSize = Seg.ZeroFillSize; |
| 163 | SI.AG = AG; |
| 164 | SI.WorkingMem = Seg.WorkingMem; |
| 165 | |
| 166 | SegInfos.push_back(x: SI); |
| 167 | } |
| 168 | |
| 169 | UsedMemory.insert(KV: {Result->Start, NextSegAddr - Result->Start}); |
| 170 | AllocPoolKey[Result->Start] = PoolKey; |
| 171 | |
| 172 | if (NextSegAddr < Result->End) { |
| 173 | // Save the remaining memory for reuse by later allocations from the same |
| 174 | // pool. |
| 175 | getAvailableMemoryLocked(Key: PoolKey).insert(a: NextSegAddr, b: Result->End - 1, |
| 176 | y: true); |
| 177 | } |
| 178 | Mutex.unlock(); |
| 179 | |
| 180 | if (auto Err = BL.apply()) { |
| 181 | OnAllocated(std::move(Err)); |
| 182 | return; |
| 183 | } |
| 184 | |
| 185 | OnAllocated(std::make_unique<InFlightAlloc>(args&: *this, args&: G, args&: Result->Start, |
| 186 | args: std::move(SegInfos))); |
| 187 | }; |
| 188 | |
| 189 | Mutex.lock(); |
| 190 | |
| 191 | // find an already reserved range in this pool that is large enough |
| 192 | ExecutorAddrRange SelectedRange{}; |
| 193 | |
| 194 | AvailableMemoryMap &Avail = getAvailableMemoryLocked(Key: PoolKey); |
| 195 | for (AvailableMemoryMap::iterator It = Avail.begin(); It != Avail.end(); |
| 196 | It++) { |
| 197 | if (It.stop() - It.start() + 1 >= TotalSize) { |
| 198 | SelectedRange = ExecutorAddrRange(It.start(), It.stop() + 1); |
| 199 | It.erase(); |
| 200 | break; |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | if (SelectedRange.empty()) { // no already reserved range was found |
| 205 | // A reservation is needed. If this pool already owns one, reserving |
| 206 | // another (possibly out-of-range) slab may place its objects out of range, |
| 207 | // so consult the growth policy first. |
| 208 | bool IsFirstReservation = !Pools[PoolKey].Reserved; |
| 209 | Pools[PoolKey].Reserved = true; |
| 210 | auto TotalAllocation = alignTo(Value: TotalSize, Align: ReservationUnits); |
| 211 | if (!IsFirstReservation) { |
| 212 | if (auto Err = OnSlabGrow(JD, TotalAllocation)) |
| 213 | return CompleteAllocation(std::move(Err)); |
| 214 | } |
| 215 | Mapper->reserve(NumBytes: TotalAllocation, |
| 216 | OnReserved: [this, PoolKey, IsFirstReservation, |
| 217 | CompleteAllocation = std::move(CompleteAllocation)]( |
| 218 | Expected<ExecutorAddrRange> Result) mutable { |
| 219 | // If this was the pool's first reservation attempt and it |
| 220 | // failed, undo the Reserved flag so a later attempt is |
| 221 | // treated as the first one again, rather than |
| 222 | // incorrectly triggering the growth policy for a pool |
| 223 | // that never got a slab. |
| 224 | if (!Result && IsFirstReservation) |
| 225 | Pools[PoolKey].Reserved = false; |
| 226 | CompleteAllocation(std::move(Result)); |
| 227 | }); |
| 228 | } else { |
| 229 | CompleteAllocation(SelectedRange); |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | void MapperJITLinkMemoryManager::deallocate( |
| 234 | std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) { |
| 235 | std::vector<ExecutorAddr> Bases; |
| 236 | Bases.reserve(n: Allocs.size()); |
| 237 | for (auto &FA : Allocs) { |
| 238 | ExecutorAddr Addr = FA.getAddress(); |
| 239 | Bases.push_back(x: Addr); |
| 240 | } |
| 241 | |
| 242 | Mapper->deinitialize(Allocations: Bases, OnDeInitialized: [this, Allocs = std::move(Allocs), |
| 243 | OnDeallocated = std::move(OnDeallocated)]( |
| 244 | llvm::Error Err) mutable { |
| 245 | // TODO: How should we treat memory that we fail to deinitialize? |
| 246 | // We're currently bailing out and treating it as "burned" -- should we |
| 247 | // require that a failure to deinitialize still reset the memory so that |
| 248 | // we can reclaim it? |
| 249 | if (Err) { |
| 250 | for (auto &FA : Allocs) |
| 251 | FA.release(); |
| 252 | OnDeallocated(std::move(Err)); |
| 253 | return; |
| 254 | } |
| 255 | |
| 256 | { |
| 257 | std::lock_guard<std::mutex> Lock(Mutex); |
| 258 | |
| 259 | for (auto &FA : Allocs) { |
| 260 | ExecutorAddr Addr = FA.getAddress(); |
| 261 | ExecutorAddrDiff Size = UsedMemory[Addr]; |
| 262 | |
| 263 | UsedMemory.erase(Val: Addr); |
| 264 | |
| 265 | // Return the range to the pool it was allocated from. |
| 266 | const JITLinkDylib *PoolKey = AllocPoolKey.lookup(Val: Addr); |
| 267 | AllocPoolKey.erase(Val: Addr); |
| 268 | getAvailableMemoryLocked(Key: PoolKey).insert(a: Addr, b: Addr + Size - 1, y: true); |
| 269 | |
| 270 | FA.release(); |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | OnDeallocated(Error::success()); |
| 275 | }); |
| 276 | } |
| 277 | |
| 278 | } // end namespace orc |
| 279 | } // end namespace llvm |
| 280 | |