1//===-- mem_map_linux.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "platform.h"
10
11#if SCUDO_LINUX
12
13#include "mem_map_linux.h"
14
15#include "common.h"
16#include "internal_defs.h"
17#include "linux.h"
18#include "mutex.h"
19#include "report_linux.h"
20#include "string_utils.h"
21
22#include <errno.h>
23#include <fcntl.h>
24#include <linux/futex.h>
25#include <sched.h>
26#include <stdio.h>
27#include <stdlib.h>
28#include <string.h>
29#include <sys/mman.h>
30#include <sys/stat.h>
31#include <sys/syscall.h>
32#include <sys/time.h>
33#include <time.h>
34#include <unistd.h>
35
36#if SCUDO_ANDROID
37// TODO(chiahungduan): Review if we still need the followings macros.
38#include <sys/prctl.h>
39// Definitions of prctl arguments to set a vma name in Android kernels.
40#define ANDROID_PR_SET_VMA 0x53564d41
41#define ANDROID_PR_SET_VMA_ANON_NAME 0
42#endif
43
44namespace scudo {
45
46static void *mmapWrapper(uptr Addr, uptr Size, const char *Name, uptr Flags) {
47 int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS;
48 int MmapProt;
49 if (Flags & MAP_NOACCESS) {
50 MmapFlags |= MAP_NORESERVE;
51 MmapProt = PROT_NONE;
52 } else {
53 MmapProt = PROT_READ | PROT_WRITE;
54 }
55#if defined(__aarch64__)
56#ifndef PROT_MTE
57#define PROT_MTE 0x20
58#endif
59 if (Flags & MAP_MEMTAG)
60 MmapProt |= PROT_MTE;
61#endif
62 if (Addr)
63 MmapFlags |= MAP_FIXED;
64 void *P =
65 mmap(addr: reinterpret_cast<void *>(Addr), len: Size, prot: MmapProt, flags: MmapFlags, fd: -1, offset: 0);
66 if (P == MAP_FAILED) {
67 if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
68 reportMapError(errno == ENOMEM ? Size : 0);
69 return nullptr;
70 }
71#if SCUDO_ANDROID
72 if (Name)
73 prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name);
74#else
75 (void)Name;
76#endif
77
78 return P;
79}
80
81bool MemMapLinux::mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags) {
82 void *P = mmapWrapper(Addr, Size, Name, Flags);
83 if (P == nullptr)
84 return false;
85
86 MapBase = reinterpret_cast<uptr>(P);
87 MapCapacity = Size;
88 return true;
89}
90
91void MemMapLinux::unmapImpl(uptr Addr, uptr Size) {
92 // If we unmap all the pages, also mark `MapBase` to 0 to indicate invalid
93 // status.
94 if (Size == MapCapacity) {
95 MapBase = MapCapacity = 0;
96 } else {
97 // This is partial unmap and is unmapping the pages from the beginning,
98 // shift `MapBase` to the new base.
99 if (MapBase == Addr)
100 MapBase = Addr + Size;
101 MapCapacity -= Size;
102 }
103
104 if (munmap(addr: reinterpret_cast<void *>(Addr), len: Size) != 0)
105 reportUnmapError(Addr, Size);
106}
107
108bool MemMapLinux::remapImpl(uptr Addr, uptr Size, const char *Name,
109 uptr Flags) {
110 void *P = mmapWrapper(Addr, Size, Name, Flags);
111 if (reinterpret_cast<uptr>(P) != Addr)
112 reportMapError();
113 return true;
114}
115
116void MemMapLinux::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) {
117 int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
118 if (mprotect(addr: reinterpret_cast<void *>(Addr), len: Size, prot: Prot) != 0)
119 reportProtectError(Addr, Size, Prot);
120}
121
122void MemMapLinux::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
123 void *Addr = reinterpret_cast<void *>(From);
124
125 int rc;
126 while ((rc = madvise(addr: Addr, len: Size, MADV_DONTNEED)) == -1 && errno == EAGAIN) {
127 }
128 if (rc == -1) {
129 // If we can't madvies the memory, then we still need to zero it.
130 memset(s: Addr, c: 0, n: Size);
131 }
132}
133
134s64 MemMapLinux::getResidentPagesImpl(uptr From, uptr Size) {
135 unsigned char PageData[256];
136
137 uptr PageSize = getPageSizeCached();
138 uptr PageSizeLog = getPageSizeLogCached();
139
140 // Make sure the address is page aligned.
141 uptr CurrentAddress = From & ~(PageSize - 1);
142 uptr LastAddress = roundUp(X: From + Size, Boundary: PageSize);
143 s64 ResidentPages = 0;
144 while (CurrentAddress < LastAddress) {
145 uptr Length = LastAddress - CurrentAddress;
146 if ((Length >> PageSizeLog) > sizeof(PageData)) {
147 Length = sizeof(PageData) << PageSizeLog;
148 }
149 if (mincore(start: reinterpret_cast<void *>(CurrentAddress), len: Length, vec: PageData) ==
150 -1) {
151 ScopedString Str;
152 Str.append(Format: "mincore failed: %s\n", strerror(errno));
153 Str.output();
154 return -1;
155 }
156 for (size_t I = 0; I < Length >> PageSizeLog; ++I) {
157 if (PageData[I])
158 ++ResidentPages;
159 }
160 CurrentAddress += Length;
161 }
162
163 return ResidentPages;
164}
165
166bool ReservedMemoryLinux::createImpl(uptr Addr, uptr Size, const char *Name,
167 uptr Flags) {
168 ReservedMemoryLinux::MemMapT MemMap;
169 if (!MemMap.map(Addr, Size, Name, Flags: Flags | MAP_NOACCESS))
170 return false;
171
172 MapBase = MemMap.getBase();
173 MapCapacity = MemMap.getCapacity();
174
175 return true;
176}
177
178void ReservedMemoryLinux::releaseImpl() {
179 if (munmap(addr: reinterpret_cast<void *>(getBase()), len: getCapacity()) != 0)
180 reportUnmapError(Addr: getBase(), Size: getCapacity());
181}
182
183ReservedMemoryLinux::MemMapT ReservedMemoryLinux::dispatchImpl(uptr Addr,
184 uptr Size) {
185 return ReservedMemoryLinux::MemMapT(Addr, Size);
186}
187
188} // namespace scudo
189
190#endif // SCUDO_LINUX
191