1//===-- mem_map_linux.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "platform.h"
10
11#if SCUDO_LINUX
12
13#include "mem_map_linux.h"
14
15#include "common.h"
16#include "internal_defs.h"
17#include "linux.h"
18#include "mutex.h"
19#include "report_linux.h"
20#include "string_utils.h"
21
22#include <errno.h>
23#include <fcntl.h>
24#include <linux/futex.h>
25#include <sched.h>
26#include <stdio.h>
27#include <stdlib.h>
28#include <string.h>
29#include <sys/mman.h>
30#include <sys/stat.h>
31#include <sys/syscall.h>
32#include <sys/time.h>
33#include <time.h>
34#include <unistd.h>
35
36#if SCUDO_ANDROID
37// TODO(chiahungduan): Review if we still need the followings macros.
38#include <sys/prctl.h>
39// Definitions of prctl arguments to set a vma name in Android kernels.
40#define ANDROID_PR_SET_VMA 0x53564d41
41#define ANDROID_PR_SET_VMA_ANON_NAME 0
42#endif
43
44namespace scudo {
45
46static void *mmapWrapper(uptr Addr, uptr Size, const char *Name, uptr Flags) {
47 int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS;
48 int MmapProt;
49 if (Flags & MAP_NOACCESS) {
50 MmapFlags |= MAP_NORESERVE;
51 MmapProt = PROT_NONE;
52 } else {
53 MmapProt = PROT_READ | PROT_WRITE;
54 }
55#if defined(__aarch64__)
56#ifndef PROT_MTE
57#define PROT_MTE 0x20
58#endif
59 if (Flags & MAP_MEMTAG)
60 MmapProt |= PROT_MTE;
61#endif
62 if (Addr)
63 MmapFlags |= MAP_FIXED;
64 void *P =
65 mmap(addr: reinterpret_cast<void *>(Addr), len: Size, prot: MmapProt, flags: MmapFlags, fd: -1, offset: 0);
66 if (P == MAP_FAILED) {
67 if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
68 reportMapError(errno == ENOMEM ? Size : 0);
69 return nullptr;
70 }
71
72 if (Addr && reinterpret_cast<uptr>(P) != Addr)
73 reportMapFixedError(ExpectedAddr: reinterpret_cast<uptr>(P), RequestedAddr: Addr);
74
75#if SCUDO_ANDROID
76 if (Name)
77 prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name);
78#else
79 (void)Name;
80#endif
81
82 return P;
83}
84
85bool MemMapLinux::mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags) {
86 void *P = mmapWrapper(Addr, Size, Name, Flags);
87 if (P == nullptr)
88 return false;
89
90 MapBase = reinterpret_cast<uptr>(P);
91 MapCapacity = Size;
92 return true;
93}
94
95void MemMapLinux::unmapImpl(uptr Addr, uptr Size) {
96 // If we unmap all the pages, also mark `MapBase` to 0 to indicate invalid
97 // status.
98 if (Size == MapCapacity) {
99 MapBase = MapCapacity = 0;
100 } else {
101 // This is partial unmap and is unmapping the pages from the beginning,
102 // shift `MapBase` to the new base.
103 if (MapBase == Addr)
104 MapBase = Addr + Size;
105 MapCapacity -= Size;
106 }
107
108 if (munmap(addr: reinterpret_cast<void *>(Addr), len: Size) != 0)
109 reportUnmapError(Addr, Size);
110}
111
112bool MemMapLinux::remapImpl(uptr Addr, uptr Size, const char *Name,
113 uptr Flags) {
114 void *P = mmapWrapper(Addr, Size, Name, Flags);
115 return reinterpret_cast<uptr>(P) == Addr;
116}
117
118void MemMapLinux::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) {
119 int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
120 if (mprotect(addr: reinterpret_cast<void *>(Addr), len: Size, prot: Prot) != 0)
121 reportProtectError(Addr, Size, Prot);
122}
123
124void MemMapLinux::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
125 void *Addr = reinterpret_cast<void *>(From);
126
127 int rc;
128 while ((rc = madvise(addr: Addr, len: Size, MADV_DONTNEED)) == -1 && errno == EAGAIN) {
129 }
130 if (rc == -1) {
131 // If we can't madvies the memory, then we still need to zero it.
132 memset(s: Addr, c: 0, n: Size);
133 }
134}
135
136s64 MemMapLinux::getResidentPagesImpl(uptr From, uptr Size) {
137 unsigned char PageData[256];
138
139 uptr PageSize = getPageSizeCached();
140 uptr PageSizeLog = getPageSizeLogCached();
141
142 // Make sure the address is page aligned.
143 uptr CurrentAddress = From & ~(PageSize - 1);
144 uptr LastAddress = roundUp(X: From + Size, Boundary: PageSize);
145 s64 ResidentPages = 0;
146 while (CurrentAddress < LastAddress) {
147 uptr Length = LastAddress - CurrentAddress;
148 if ((Length >> PageSizeLog) > sizeof(PageData)) {
149 Length = sizeof(PageData) << PageSizeLog;
150 }
151 if (mincore(start: reinterpret_cast<void *>(CurrentAddress), len: Length, vec: PageData) ==
152 -1) {
153 ScopedString Str;
154 Str.append(Format: "mincore failed: %s\n", strerror(errno));
155 Str.output();
156 return -1;
157 }
158 for (size_t I = 0; I < Length >> PageSizeLog; ++I) {
159 if (PageData[I])
160 ++ResidentPages;
161 }
162 CurrentAddress += Length;
163 }
164
165 return ResidentPages;
166}
167
168bool ReservedMemoryLinux::createImpl(uptr Addr, uptr Size, const char *Name,
169 uptr Flags) {
170 ReservedMemoryLinux::MemMapT MemMap;
171 if (!MemMap.map(Addr, Size, Name, Flags: Flags | MAP_NOACCESS))
172 return false;
173
174 MapBase = MemMap.getBase();
175 MapCapacity = MemMap.getCapacity();
176
177 return true;
178}
179
180void ReservedMemoryLinux::releaseImpl() {
181 if (munmap(addr: reinterpret_cast<void *>(getBase()), len: getCapacity()) != 0)
182 reportUnmapError(Addr: getBase(), Size: getCapacity());
183}
184
185ReservedMemoryLinux::MemMapT ReservedMemoryLinux::dispatchImpl(uptr Addr,
186 uptr Size) {
187 return ReservedMemoryLinux::MemMapT(Addr, Size);
188}
189
190} // namespace scudo
191
192#endif // SCUDO_LINUX
193