1//===-- linux.cpp -----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "platform.h"
10
11#if SCUDO_LINUX
12
13#include "common.h"
14#include "internal_defs.h"
15#include "linux.h"
16#include "mutex.h"
17#include "report_linux.h"
18#include "string_utils.h"
19
20#include <errno.h>
21#include <fcntl.h>
22#include <linux/futex.h>
23#include <sched.h>
24#include <stdio.h>
25#include <stdlib.h>
26#include <string.h>
27#include <sys/mman.h>
28#include <sys/stat.h>
29#include <sys/syscall.h>
30#include <sys/time.h>
31#include <time.h>
32#include <unistd.h>
33
34#if SCUDO_ANDROID
35#include <sys/prctl.h>
36// Definitions of prctl arguments to set a vma name in Android kernels.
37#define ANDROID_PR_SET_VMA 0x53564d41
38#define ANDROID_PR_SET_VMA_ANON_NAME 0
39#endif
40
41namespace scudo {
42
43#if !defined(SCUDO_PAGE_SIZE)
44// This function is only used when page size is not hard-coded.
45uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
46#endif
47
48void NORETURN die() { abort(); }
49
50// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
51void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
52 UNUSED MapPlatformData *Data) {
53 int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS;
54 int MmapProt;
55 if (Flags & MAP_NOACCESS) {
56 MmapFlags |= MAP_NORESERVE;
57 MmapProt = PROT_NONE;
58 } else {
59 MmapProt = PROT_READ | PROT_WRITE;
60 }
61#if defined(__aarch64__)
62#ifndef PROT_MTE
63#define PROT_MTE 0x20
64#endif
65 if (Flags & MAP_MEMTAG)
66 MmapProt |= PROT_MTE;
67#endif
68 if (Addr)
69 MmapFlags |= MAP_FIXED;
70 void *P = mmap(addr: Addr, len: Size, prot: MmapProt, flags: MmapFlags, fd: -1, offset: 0);
71 if (P == MAP_FAILED) {
72 if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
73 reportMapError(errno == ENOMEM ? Size : 0);
74 return nullptr;
75 }
76#if SCUDO_ANDROID
77 if (Name)
78 prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name);
79#endif
80 return P;
81}
82
83// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
84void unmap(void *Addr, uptr Size, UNUSED uptr Flags,
85 UNUSED MapPlatformData *Data) {
86 if (munmap(addr: Addr, len: Size) != 0)
87 reportUnmapError(Addr: reinterpret_cast<uptr>(Addr), Size);
88}
89
90// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
91void setMemoryPermission(uptr Addr, uptr Size, uptr Flags,
92 UNUSED MapPlatformData *Data) {
93 int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
94 if (mprotect(addr: reinterpret_cast<void *>(Addr), len: Size, prot: Prot) != 0)
95 reportProtectError(Addr, Size, Prot);
96}
97
98// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
99void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
100 UNUSED MapPlatformData *Data) {
101 void *Addr = reinterpret_cast<void *>(BaseAddress + Offset);
102
103 while (madvise(addr: Addr, len: Size, MADV_DONTNEED) == -1 && errno == EAGAIN) {
104 }
105}
106
107// Calling getenv should be fine (c)(tm) at any time.
108const char *getEnv(const char *Name) { return getenv(name: Name); }
109
110namespace {
111enum State : u32 { Unlocked = 0, Locked = 1, Sleeping = 2 };
112}
113
114bool HybridMutex::tryLock() {
115 return atomic_compare_exchange_strong(A: &M, Cmp: Unlocked, Xchg: Locked,
116 MO: memory_order_acquire) == Unlocked;
117}
118
119// The following is based on https://akkadia.org/drepper/futex.pdf.
120void HybridMutex::lockSlow() {
121 u32 V = atomic_compare_exchange_strong(A: &M, Cmp: Unlocked, Xchg: Locked,
122 MO: memory_order_acquire);
123 if (V == Unlocked)
124 return;
125 if (V != Sleeping)
126 V = atomic_exchange(A: &M, V: Sleeping, MO: memory_order_acquire);
127 while (V != Unlocked) {
128 syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAIT_PRIVATE, Sleeping,
129 nullptr, nullptr, 0);
130 V = atomic_exchange(A: &M, V: Sleeping, MO: memory_order_acquire);
131 }
132}
133
134void HybridMutex::unlock() {
135 if (atomic_fetch_sub(A: &M, V: 1U, MO: memory_order_release) != Locked) {
136 atomic_store(A: &M, V: Unlocked, MO: memory_order_release);
137 syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAKE_PRIVATE, 1,
138 nullptr, nullptr, 0);
139 }
140}
141
142void HybridMutex::assertHeldImpl() {
143 CHECK(atomic_load(&M, memory_order_acquire) != Unlocked);
144}
145
146u64 getMonotonicTime() {
147 timespec TS;
148 clock_gettime(CLOCK_MONOTONIC, tp: &TS);
149 return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) +
150 static_cast<u64>(TS.tv_nsec);
151}
152
153u64 getMonotonicTimeFast() {
154#if defined(CLOCK_MONOTONIC_COARSE)
155 timespec TS;
156 clock_gettime(CLOCK_MONOTONIC_COARSE, tp: &TS);
157 return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) +
158 static_cast<u64>(TS.tv_nsec);
159#else
160 return getMonotonicTime();
161#endif
162}
163
164u32 getNumberOfCPUs() {
165 cpu_set_t CPUs;
166 // sched_getaffinity can fail for a variety of legitimate reasons (lack of
167 // CAP_SYS_NICE, syscall filtering, etc), in which case we shall return 0.
168 if (sched_getaffinity(pid: 0, cpusetsize: sizeof(cpu_set_t), cpuset: &CPUs) != 0)
169 return 0;
170 return static_cast<u32>(CPU_COUNT(&CPUs));
171}
172
173u32 getThreadID() {
174#if SCUDO_ANDROID
175 return static_cast<u32>(gettid());
176#else
177 return static_cast<u32>(syscall(SYS_gettid));
178#endif
179}
180
181// Blocking is possibly unused if the getrandom block is not compiled in.
182bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) {
183 if (!Buffer || !Length || Length > MaxRandomLength)
184 return false;
185 ssize_t ReadBytes;
186#if defined(SYS_getrandom)
187#if !defined(GRND_NONBLOCK)
188#define GRND_NONBLOCK 1
189#endif
190 // Up to 256 bytes, getrandom will not be interrupted.
191 ReadBytes =
192 syscall(SYS_getrandom, Buffer, Length, Blocking ? 0 : GRND_NONBLOCK);
193 if (ReadBytes == static_cast<ssize_t>(Length))
194 return true;
195 // If this system call is not implemented in the kernel, then we will try
196 // and use /dev/urandom. Otherwise, if the syscall fails, return false
197 // assuming that trying to read /dev/urandom will cause a delay waiting for
198 // the random data to be usable.
199 if (errno != ENOSYS)
200 return false;
201#endif // defined(SYS_getrandom)
202 // Up to 256 bytes, a read off /dev/urandom will not be interrupted.
203 // Blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
204 const int FileDesc = open(file: "/dev/urandom", O_RDONLY);
205 if (FileDesc == -1)
206 return false;
207 ReadBytes = read(fd: FileDesc, buf: Buffer, nbytes: Length);
208 close(fd: FileDesc);
209 return (ReadBytes == static_cast<ssize_t>(Length));
210}
211
212// Allocation free syslog-like API.
213extern "C" WEAK int async_safe_write_log(int pri, const char *tag,
214 const char *msg);
215
216void outputRaw(const char *Buffer) {
217 if (&async_safe_write_log) {
218 constexpr s32 AndroidLogInfo = 4;
219 constexpr uptr MaxLength = 1024U;
220 char LocalBuffer[MaxLength];
221 while (strlen(s: Buffer) > MaxLength) {
222 uptr P;
223 for (P = MaxLength - 1; P > 0; P--) {
224 if (Buffer[P] == '\n') {
225 memcpy(dest: LocalBuffer, src: Buffer, n: P);
226 LocalBuffer[P] = '\0';
227 async_safe_write_log(pri: AndroidLogInfo, tag: "scudo", msg: LocalBuffer);
228 Buffer = &Buffer[P + 1];
229 break;
230 }
231 }
232 // If no newline was found, just log the buffer.
233 if (P == 0)
234 break;
235 }
236 async_safe_write_log(pri: AndroidLogInfo, tag: "scudo", msg: Buffer);
237 } else {
238 (void)write(fd: 2, buf: Buffer, n: strlen(s: Buffer));
239 }
240}
241
242extern "C" WEAK void android_set_abort_message(const char *);
243
244void setAbortMessage(const char *Message) {
245 if (&android_set_abort_message)
246 android_set_abort_message(Message);
247}
248
249u64 getResidentPages(uptr BaseAddress, uptr Size) {
250 unsigned char PageData[256];
251
252 uptr PageSize = getPageSizeCached();
253 uptr PageSizeLog = getPageSizeLogCached();
254
255 // Make sure the address is page aligned.
256 uptr CurrentAddress = BaseAddress & ~(PageSize - 1);
257 uptr LastAddress = roundUp(X: BaseAddress + Size, Boundary: PageSize);
258 u64 ResidentPages = 0;
259 while (CurrentAddress < LastAddress) {
260 uptr Length = LastAddress - CurrentAddress;
261 if ((Length >> PageSizeLog) > sizeof(PageData)) {
262 Length = sizeof(PageData) << PageSizeLog;
263 }
264 if (mincore(start: reinterpret_cast<void *>(CurrentAddress), len: Length, vec: PageData) ==
265 -1) {
266 ScopedString Str;
267 Str.append(Format: "mincore failed: %s\n", strerror(errno));
268 Str.output();
269 break;
270 }
271 for (size_t I = 0; I < Length >> PageSizeLog; ++I) {
272 if (PageData[I])
273 ++ResidentPages;
274 }
275 CurrentAddress += Length;
276 }
277
278 return ResidentPages;
279}
280
281} // namespace scudo
282
283#endif // SCUDO_LINUX
284