1//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the section-based memory manager used by the MCJIT
10// execution engine and RuntimeDyld
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ExecutionEngine/SectionMemoryManager.h"
15#include "llvm/Config/config.h"
16#include "llvm/Support/Process.h"
17
18namespace llvm {
19
20uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
21 unsigned Alignment,
22 unsigned SectionID,
23 StringRef SectionName,
24 bool IsReadOnly) {
25 if (IsReadOnly)
26 return allocateSection(Purpose: SectionMemoryManager::AllocationPurpose::ROData,
27 Size, Alignment);
28 return allocateSection(Purpose: SectionMemoryManager::AllocationPurpose::RWData, Size,
29 Alignment);
30}
31
32uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
33 unsigned Alignment,
34 unsigned SectionID,
35 StringRef SectionName) {
36 return allocateSection(Purpose: SectionMemoryManager::AllocationPurpose::Code, Size,
37 Alignment);
38}
39
40uint8_t *SectionMemoryManager::allocateSection(
41 SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
42 unsigned Alignment) {
43 if (!Alignment)
44 Alignment = 16;
45
46 assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
47
48 uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
49 uintptr_t Addr = 0;
50
51 MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
52 switch (Purpose) {
53 case AllocationPurpose::Code:
54 return CodeMem;
55 case AllocationPurpose::ROData:
56 return RODataMem;
57 case AllocationPurpose::RWData:
58 return RWDataMem;
59 }
60 llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
61 }();
62
63 // Look in the list of free memory regions and use a block there if one
64 // is available.
65 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
66 if (FreeMB.Free.allocatedSize() >= RequiredSize) {
67 Addr = (uintptr_t)FreeMB.Free.base();
68 uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();
69 // Align the address.
70 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
71
72 if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
73 // The part of the block we're giving out to the user is now pending
74 MemGroup.PendingMem.push_back(Elt: sys::MemoryBlock((void *)Addr, Size));
75
76 // Remember this pending block, such that future allocations can just
77 // modify it rather than creating a new one
78 FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
79 } else {
80 sys::MemoryBlock &PendingMB =
81 MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
82 PendingMB = sys::MemoryBlock(PendingMB.base(),
83 Addr + Size - (uintptr_t)PendingMB.base());
84 }
85
86 // Remember how much free space is now left in this block
87 FreeMB.Free =
88 sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
89 return (uint8_t *)Addr;
90 }
91 }
92
93 // No pre-allocated free block was large enough. Allocate a new memory region.
94 // Note that all sections get allocated as read-write. The permissions will
95 // be updated later based on memory group.
96 //
97 // FIXME: It would be useful to define a default allocation size (or add
98 // it as a constructor parameter) to minimize the number of allocations.
99 //
100 // FIXME: Initialize the Near member for each memory group to avoid
101 // interleaving.
102 std::error_code ec;
103 sys::MemoryBlock MB = MMapper->allocateMappedMemory(
104 Purpose, NumBytes: RequiredSize, NearBlock: &MemGroup.Near,
105 Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC&: ec);
106 if (ec) {
107 // FIXME: Add error propagation to the interface.
108 return nullptr;
109 }
110
111 // Save this address as the basis for our next request
112 MemGroup.Near = MB;
113
114 // Copy the address to all the other groups, if they have not
115 // been initialized.
116 if (CodeMem.Near.base() == nullptr)
117 CodeMem.Near = MB;
118 if (RODataMem.Near.base() == nullptr)
119 RODataMem.Near = MB;
120 if (RWDataMem.Near.base() == nullptr)
121 RWDataMem.Near = MB;
122
123 // Remember that we allocated this memory
124 MemGroup.AllocatedMem.push_back(Elt: MB);
125 Addr = (uintptr_t)MB.base();
126 uintptr_t EndOfBlock = Addr + MB.allocatedSize();
127
128 // Align the address.
129 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
130
131 // The part of the block we're giving out to the user is now pending
132 MemGroup.PendingMem.push_back(Elt: sys::MemoryBlock((void *)Addr, Size));
133
134 // The allocateMappedMemory may allocate much more memory than we need. In
135 // this case, we store the unused memory as a free memory block.
136 unsigned FreeSize = EndOfBlock - Addr - Size;
137 if (FreeSize > 16) {
138 FreeMemBlock FreeMB;
139 FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
140 FreeMB.PendingPrefixIndex = (unsigned)-1;
141 MemGroup.FreeMem.push_back(Elt: FreeMB);
142 }
143
144 // Return aligned address
145 return (uint8_t *)Addr;
146}
147
148bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
149 // FIXME: Should in-progress permissions be reverted if an error occurs?
150 std::error_code ec;
151
152 // Make code memory executable.
153 ec = applyMemoryGroupPermissions(MemGroup&: CodeMem,
154 Permissions: sys::Memory::MF_READ | sys::Memory::MF_EXEC);
155 if (ec) {
156 if (ErrMsg) {
157 *ErrMsg = ec.message();
158 }
159 return true;
160 }
161
162 // Make read-only data memory read-only.
163 ec = applyMemoryGroupPermissions(MemGroup&: RODataMem, Permissions: sys::Memory::MF_READ);
164 if (ec) {
165 if (ErrMsg) {
166 *ErrMsg = ec.message();
167 }
168 return true;
169 }
170
171 // Read-write data memory already has the correct permissions
172
173 // Some platforms with separate data cache and instruction cache require
174 // explicit cache flush, otherwise JIT code manipulations (like resolved
175 // relocations) will get to the data cache but not to the instruction cache.
176 invalidateInstructionCache();
177
178 return false;
179}
180
181static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
182 static const size_t PageSize = sys::Process::getPageSizeEstimate();
183
184 size_t StartOverlap =
185 (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
186
187 size_t TrimmedSize = M.allocatedSize();
188 TrimmedSize -= StartOverlap;
189 TrimmedSize -= TrimmedSize % PageSize;
190
191 sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
192 TrimmedSize);
193
194 assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
195 assert((Trimmed.allocatedSize() % PageSize) == 0);
196 assert(M.base() <= Trimmed.base() &&
197 Trimmed.allocatedSize() <= M.allocatedSize());
198
199 return Trimmed;
200}
201
202std::error_code
203SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
204 unsigned Permissions) {
205 for (sys::MemoryBlock &MB : MemGroup.PendingMem)
206 if (std::error_code EC = MMapper->protectMappedMemory(Block: MB, Flags: Permissions))
207 return EC;
208
209 MemGroup.PendingMem.clear();
210
211 // Now go through free blocks and trim any of them that don't span the entire
212 // page because one of the pending blocks may have overlapped it.
213 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
214 FreeMB.Free = trimBlockToPageSize(M: FreeMB.Free);
215 // We cleared the PendingMem list, so all these pointers are now invalid
216 FreeMB.PendingPrefixIndex = (unsigned)-1;
217 }
218
219 // Remove all blocks which are now empty
220 erase_if(C&: MemGroup.FreeMem, P: [](FreeMemBlock &FreeMB) {
221 return FreeMB.Free.allocatedSize() == 0;
222 });
223
224 return std::error_code();
225}
226
227void SectionMemoryManager::invalidateInstructionCache() {
228 for (sys::MemoryBlock &Block : CodeMem.PendingMem)
229 sys::Memory::InvalidateInstructionCache(Addr: Block.base(),
230 Len: Block.allocatedSize());
231}
232
233SectionMemoryManager::~SectionMemoryManager() {
234 for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
235 for (sys::MemoryBlock &Block : Group->AllocatedMem)
236 MMapper->releaseMappedMemory(M&: Block);
237 }
238}
239
240SectionMemoryManager::MemoryMapper::~MemoryMapper() = default;
241
242void SectionMemoryManager::anchor() {}
243
244namespace {
245// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
246// into sys::Memory.
247class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
248public:
249 sys::MemoryBlock
250 allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
251 size_t NumBytes, const sys::MemoryBlock *const NearBlock,
252 unsigned Flags, std::error_code &EC) override {
253 return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
254 }
255
256 std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
257 unsigned Flags) override {
258 return sys::Memory::protectMappedMemory(Block, Flags);
259 }
260
261 std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
262 return sys::Memory::releaseMappedMemory(Block&: M);
263 }
264};
265} // namespace
266
267SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM)
268 : MMapper(UnownedMM), OwnedMMapper(nullptr) {
269 if (!MMapper) {
270 OwnedMMapper = std::make_unique<DefaultMMapper>();
271 MMapper = OwnedMMapper.get();
272 }
273}
274
275} // namespace llvm
276