1//===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Platform-specific code.
12//===----------------------------------------------------------------------===//
13
14#ifndef TSAN_PLATFORM_H
15#define TSAN_PLATFORM_H
16
17#if !defined(__LP64__) && !defined(_WIN64)
18# error "Only 64-bit is supported"
19#endif
20
21#include "sanitizer_common/sanitizer_common.h"
22#include "tsan_defs.h"
23
24namespace __tsan {
25
26enum {
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping = 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping = 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity = 1 << 2,
37 // Meta for an app region overlaps with the meta of another app region.
38 // This is determined by recomputing the individual meta regions for
39 // each app region.
40 //
41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42 // is no MetaToMem function. However, note that (!kBrokenLinearity
43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44 kBrokenAliasedMetas = 1 << 3,
45};
46
47/*
48C/C++ on linux/x86_64 and freebsd/x86_64
490000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
500200 0000 0000 - 1000 0000 0000: -
511000 0000 0000 - 3000 0000 0000: shadow (32TB)
523000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
533800 0000 0000 - 5500 0000 0000: -
545500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
555a00 0000 0000 - 7200 0000 0000: -
567200 0000 0000 - 7300 0000 0000: heap (1TB)
577300 0000 0000 - 7a00 0000 0000: -
587a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
59
60C/C++ on netbsd/amd64 can reuse the same mapping:
61 * The address space starts from 0x1000 (option with 0x0) and ends with
62 0x7f7ffffff000.
63 * LoAppMem-kHeapMemEnd can be reused as it is.
64 * No VDSO support.
65 * No MidAppMem region.
66 * No additional HeapMem region.
67 * HiAppMem contains the stack, loader, shared libraries and heap.
68 * Stack on NetBSD/amd64 has prereserved 128MB.
69 * Heap grows downwards (top-down).
70 * ASLR must be disabled per-process or globally.
71*/
72struct Mapping48AddressSpace {
73 static const uptr kMetaShadowBeg = 0x300000000000ull;
74 static const uptr kMetaShadowEnd = 0x380000000000ull;
75 static const uptr kShadowBeg = 0x100000000000ull;
76 static const uptr kShadowEnd = 0x300000000000ull;
77 static const uptr kHeapMemBeg = 0x720000000000ull;
78 static const uptr kHeapMemEnd = 0x730000000000ull;
79 static const uptr kLoAppMemBeg = 0x000000001000ull;
80 static const uptr kLoAppMemEnd = 0x020000000000ull;
81 static const uptr kMidAppMemBeg = 0x550000000000ull;
82 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
83 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
84 static const uptr kHiAppMemEnd = 0x800000000000ull;
85 static const uptr kShadowMsk = 0x700000000000ull;
86 static const uptr kShadowXor = 0x000000000000ull;
87 static const uptr kShadowAdd = 0x100000000000ull;
88 static const uptr kVdsoBeg = 0xf000000000000000ull;
89};
90
91/*
92C/C++ on linux/mips64 (40-bit VMA)
930000 0000 00 - 0100 0000 00: - (4 GB)
940100 0000 00 - 0200 0000 00: main binary (4 GB)
950200 0000 00 - 1200 0000 00: - (64 GB)
961200 0000 00 - 2200 0000 00: shadow (64 GB)
972200 0000 00 - 4000 0000 00: - (120 GB)
984000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
995000 0000 00 - aa00 0000 00: - (360 GB)
100aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
101ab00 0000 00 - fe00 0000 00: - (332 GB)
102fe00 0000 00 - ff00 0000 00: heap (4 GB)
103ff00 0000 00 - ff80 0000 00: - (2 GB)
104ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
105*/
106struct MappingMips64_40 {
107 static const uptr kMetaShadowBeg = 0x4000000000ull;
108 static const uptr kMetaShadowEnd = 0x5000000000ull;
109 static const uptr kShadowBeg = 0x1200000000ull;
110 static const uptr kShadowEnd = 0x2200000000ull;
111 static const uptr kHeapMemBeg = 0xfe00000000ull;
112 static const uptr kHeapMemEnd = 0xff00000000ull;
113 static const uptr kLoAppMemBeg = 0x0100000000ull;
114 static const uptr kLoAppMemEnd = 0x0200000000ull;
115 static const uptr kMidAppMemBeg = 0xaa00000000ull;
116 static const uptr kMidAppMemEnd = 0xab00000000ull;
117 static const uptr kHiAppMemBeg = 0xff80000000ull;
118 static const uptr kHiAppMemEnd = 0xffffffffffull;
119 static const uptr kShadowMsk = 0xf800000000ull;
120 static const uptr kShadowXor = 0x0800000000ull;
121 static const uptr kShadowAdd = 0x0000000000ull;
122 static const uptr kVdsoBeg = 0xfffff00000ull;
123};
124
125/*
126C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
1270000 0000 00 - 0100 0000 00: - (4 GB)
1280100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
1290200 0000 00 - 0300 0000 00: heap (4 GB)
1300300 0000 00 - 0400 0000 00: - (4 GB)
1310400 0000 00 - 0800 0000 00: shadow memory (16 GB)
1320800 0000 00 - 0d00 0000 00: - (20 GB)
1330d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
1340e00 0000 00 - 1000 0000 00: -
135*/
136struct MappingAppleAarch64 {
137 static const uptr kLoAppMemBeg = 0x0100000000ull;
138 static const uptr kLoAppMemEnd = 0x0200000000ull;
139 static const uptr kHeapMemBeg = 0x0200000000ull;
140 static const uptr kHeapMemEnd = 0x0300000000ull;
141 static const uptr kShadowBeg = 0x0400000000ull;
142 static const uptr kShadowEnd = 0x0800000000ull;
143 static const uptr kMetaShadowBeg = 0x0d00000000ull;
144 static const uptr kMetaShadowEnd = 0x0e00000000ull;
145 static const uptr kHiAppMemBeg = 0x0fc0000000ull;
146 static const uptr kHiAppMemEnd = 0x0fc0000000ull;
147 static const uptr kShadowMsk = 0x0ull;
148 static const uptr kShadowXor = 0x0ull;
149 static const uptr kShadowAdd = 0x0200000000ull;
150 static const uptr kVdsoBeg = 0x7000000000000000ull;
151 static const uptr kMidAppMemBeg = 0;
152 static const uptr kMidAppMemEnd = 0;
153};
154
155/*
156C/C++ on linux/aarch64 (39-bit VMA)
1570000 0010 00 - 0500 0000 00: main binary (20 GB)
1580100 0000 00 - 2000 0000 00: -
1592000 0000 00 - 4000 0000 00: shadow memory (128 GB)
1604000 0000 00 - 4800 0000 00: metainfo (32 GB)
1614800 0000 00 - 5500 0000 00: -
1625500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
1635600 0000 00 - 7c00 0000 00: -
1647a00 0000 00 - 7d00 0000 00: heap (12 GB)
1657d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
166*/
167struct MappingAarch64_39 {
168 static const uptr kLoAppMemBeg = 0x0000001000ull;
169 static const uptr kLoAppMemEnd = 0x0500000000ull;
170 static const uptr kShadowBeg = 0x2000000000ull;
171 static const uptr kShadowEnd = 0x4000000000ull;
172 static const uptr kMetaShadowBeg = 0x4000000000ull;
173 static const uptr kMetaShadowEnd = 0x4800000000ull;
174 static const uptr kMidAppMemBeg = 0x5500000000ull;
175 static const uptr kMidAppMemEnd = 0x5a00000000ull;
176 static const uptr kHeapMemBeg = 0x7a00000000ull;
177 static const uptr kHeapMemEnd = 0x7d00000000ull;
178 static const uptr kHiAppMemBeg = 0x7d00000000ull;
179 static const uptr kHiAppMemEnd = 0x7fffffffffull;
180 static const uptr kShadowMsk = 0x7000000000ull;
181 static const uptr kShadowXor = 0x1000000000ull;
182 static const uptr kShadowAdd = 0x0000000000ull;
183 static const uptr kVdsoBeg = 0x7f00000000ull;
184};
185
186/*
187C/C++ on linux/aarch64 (42-bit VMA)
18800000 0010 00 - 02000 0000 00: main binary (128 GB)
18902000 0000 00 - 08000 0000 00: -
19010000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
19120000 0000 00 - 24000 0000 00: metainfo (256 GB)
19224000 0000 00 - 2aa00 0000 00: -
1932aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
1942c000 0000 00 - 3c000 0000 00: -
1953c000 0000 00 - 3f000 0000 00: heap (192 GB)
1963f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
197*/
198struct MappingAarch64_42 {
199 static const uptr kLoAppMemBeg = 0x00000001000ull;
200 static const uptr kLoAppMemEnd = 0x02000000000ull;
201 static const uptr kShadowBeg = 0x10000000000ull;
202 static const uptr kShadowEnd = 0x20000000000ull;
203 static const uptr kMetaShadowBeg = 0x20000000000ull;
204 static const uptr kMetaShadowEnd = 0x24000000000ull;
205 static const uptr kMidAppMemBeg = 0x2aa00000000ull;
206 static const uptr kMidAppMemEnd = 0x2c000000000ull;
207 static const uptr kHeapMemBeg = 0x3c000000000ull;
208 static const uptr kHeapMemEnd = 0x3f000000000ull;
209 static const uptr kHiAppMemBeg = 0x3f000000000ull;
210 static const uptr kHiAppMemEnd = 0x3ffffffffffull;
211 static const uptr kShadowMsk = 0x38000000000ull;
212 static const uptr kShadowXor = 0x08000000000ull;
213 static const uptr kShadowAdd = 0x00000000000ull;
214 static const uptr kVdsoBeg = 0x37f00000000ull;
215};
216
217/*
218C/C++ on linux/aarch64 (47-bit VMA)
2190000 0000 1000 - 0400 0000 0000: main binary (4096 GB)
2200400 0000 0000 - 0aaa 0000 0000: -
2210aaa 0000 0000 - 2800 0000 0000: shadow memory (30040 GB)
2222800 0000 0000 - 4000 0000 0000: -
2234000 0000 0000 - 5000 0000 0000: metainfo (16384 GB)
2245000 0000 0000 - 5555 0000 0000: -
2255555 0000 0000 - 5c00 0000 0000: main binary (PIE) (6828 GB)
2265c00 0000 0000 - 7c00 0000 0000: -
2277c00 0000 0000 - 7fff ffff ffff: modules and main thread stack (4096 GB)
228*/
229struct MappingAarch64_47 {
230 static const uptr kLoAppMemBeg = 0x0000000001000ull;
231 static const uptr kLoAppMemEnd = 0x0040000000000ull;
232 static const uptr kShadowBeg = 0x00aaa00000000ull;
233 static const uptr kShadowEnd = 0x0280000000000ull;
234 static const uptr kMetaShadowBeg = 0x0400000000000ull;
235 static const uptr kMetaShadowEnd = 0x0500000000000ull;
236 static const uptr kMidAppMemBeg = 0x0555500000000ull;
237 static const uptr kMidAppMemEnd = 0x05c0000000000ull;
238 static const uptr kHiAppMemBeg = 0x07c0000000000ull;
239 static const uptr kHiAppMemEnd = 0x0800000000000ull;
240 static const uptr kHeapMemBeg = 0x07c0000000000ull;
241 static const uptr kHeapMemEnd = 0x07c0000000000ull;
242 static const uptr kShadowMsk = 0x0600000000000ull;
243 static const uptr kShadowXor = 0x0100000000000ull;
244 static const uptr kShadowAdd = 0x0000000000000ull;
245 static const uptr kVdsoBeg = 0x07fff00000000ull;
246};
247
248/*
249C/C++ on linux/aarch64 (48-bit VMA)
2500000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
2510a00 0000 1000 - 1554 0000 0000: -
2521554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
2535400 0000 1000 - 8000 0000 0000: -
2548000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
255a000 0000 1000 - aaaa 0000 0000: -
256aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
257ac00 0000 1000 - fc00 0000 0000: -
258fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
259
260N.B. the shadow memory region has a strange start address, because it
261contains the shadows for the mid, high and low app regions (in this
262unusual order).
263*/
264struct MappingAarch64_48 {
265 static const uptr kLoAppMemBeg = 0x0000000001000ull;
266 static const uptr kLoAppMemEnd = 0x00a0000000000ull;
267 static const uptr kShadowBeg = 0x0155400000000ull;
268 static const uptr kShadowEnd = 0x0540000000000ull;
269 static const uptr kMetaShadowBeg = 0x0800000000000ull;
270 static const uptr kMetaShadowEnd = 0x0a00000000000ull;
271 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
272 static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
273 static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
274 static const uptr kHiAppMemEnd = 0x1000000000000ull;
275 static const uptr kHeapMemBeg = 0x0fc0000000000ull;
276 static const uptr kHeapMemEnd = 0x0fc0000000000ull;
277 static const uptr kShadowMsk = 0x0c00000000000ull;
278 static const uptr kShadowXor = 0x0200000000000ull;
279 static const uptr kShadowAdd = 0x0000000000000ull;
280 static const uptr kVdsoBeg = 0xffff000000000ull;
281};
282
283/* C/C++ on linux/loongarch64 (47-bit VMA)
2840000 0000 4000 - 0080 0000 0000: main binary
2850080 0000 0000 - 0100 0000 0000: -
2860100 0000 0000 - 1000 0000 0000: shadow memory
2871000 0000 0000 - 3000 0000 0000: -
2883000 0000 0000 - 3400 0000 0000: metainfo
2893400 0000 0000 - 5555 0000 0000: -
2905555 0000 0000 - 5556 0000 0000: main binary (PIE)
2915556 0000 0000 - 7ffe 0000 0000: -
2927ffe 0000 0000 - 7fff 0000 0000: heap
2937fff 0000 0000 - 7fff 8000 0000: -
2947fff 8000 0000 - 8000 0000 0000: modules and main thread stack
295*/
296struct MappingLoongArch64_47 {
297 static const uptr kMetaShadowBeg = 0x300000000000ull;
298 static const uptr kMetaShadowEnd = 0x340000000000ull;
299 static const uptr kShadowBeg = 0x010000000000ull;
300 static const uptr kShadowEnd = 0x100000000000ull;
301 static const uptr kHeapMemBeg = 0x7ffe00000000ull;
302 static const uptr kHeapMemEnd = 0x7fff00000000ull;
303 static const uptr kLoAppMemBeg = 0x000000004000ull;
304 static const uptr kLoAppMemEnd = 0x008000000000ull;
305 static const uptr kMidAppMemBeg = 0x555500000000ull;
306 static const uptr kMidAppMemEnd = 0x555600000000ull;
307 static const uptr kHiAppMemBeg = 0x7fff80000000ull;
308 static const uptr kHiAppMemEnd = 0x800000000000ull;
309 static const uptr kShadowMsk = 0x780000000000ull;
310 static const uptr kShadowXor = 0x040000000000ull;
311 static const uptr kShadowAdd = 0x000000000000ull;
312 static const uptr kVdsoBeg = 0x7fffffffc000ull;
313};
314
315/*
316C/C++ on linux/powerpc64 (44-bit VMA)
3170000 0000 0100 - 0001 0000 0000: main binary
3180001 0000 0000 - 0001 0000 0000: -
3190001 0000 0000 - 0b00 0000 0000: shadow
3200b00 0000 0000 - 0b00 0000 0000: -
3210b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
3220d00 0000 0000 - 0f00 0000 0000: -
3230f00 0000 0000 - 0f50 0000 0000: heap
3240f50 0000 0000 - 0f60 0000 0000: -
3250f60 0000 0000 - 1000 0000 0000: modules and main thread stack
326*/
327struct MappingPPC64_44 {
328 static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
329 kBrokenLinearity | kBrokenAliasedMetas;
330 static const uptr kMetaShadowBeg = 0x0b0000000000ull;
331 static const uptr kMetaShadowEnd = 0x0d0000000000ull;
332 static const uptr kShadowBeg = 0x000100000000ull;
333 static const uptr kShadowEnd = 0x0b0000000000ull;
334 static const uptr kLoAppMemBeg = 0x000000000100ull;
335 static const uptr kLoAppMemEnd = 0x000100000000ull;
336 static const uptr kHeapMemBeg = 0x0f0000000000ull;
337 static const uptr kHeapMemEnd = 0x0f5000000000ull;
338 static const uptr kHiAppMemBeg = 0x0f6000000000ull;
339 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
340 static const uptr kShadowMsk = 0x0f0000000000ull;
341 static const uptr kShadowXor = 0x002100000000ull;
342 static const uptr kShadowAdd = 0x000000000000ull;
343 static const uptr kVdsoBeg = 0x3c0000000000000ull;
344 static const uptr kMidAppMemBeg = 0;
345 static const uptr kMidAppMemEnd = 0;
346};
347
348/*
349C/C++ on linux/powerpc64 (46-bit VMA)
3500000 0000 1000 - 0100 0000 0000: main binary
3510100 0000 0000 - 0200 0000 0000: -
3520100 0000 0000 - 0800 0000 0000: shadow
3530800 0000 0000 - 1000 0000 0000: -
3541000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3551200 0000 0000 - 3d00 0000 0000: -
3563d00 0000 0000 - 3e00 0000 0000: heap
3573e00 0000 0000 - 3e80 0000 0000: -
3583e80 0000 0000 - 4000 0000 0000: modules and main thread stack
359*/
360struct MappingPPC64_46 {
361 static const uptr kMetaShadowBeg = 0x100000000000ull;
362 static const uptr kMetaShadowEnd = 0x120000000000ull;
363 static const uptr kShadowBeg = 0x010000000000ull;
364 static const uptr kShadowEnd = 0x080000000000ull;
365 static const uptr kHeapMemBeg = 0x3d0000000000ull;
366 static const uptr kHeapMemEnd = 0x3e0000000000ull;
367 static const uptr kLoAppMemBeg = 0x000000001000ull;
368 static const uptr kLoAppMemEnd = 0x010000000000ull;
369 static const uptr kHiAppMemBeg = 0x3e8000000000ull;
370 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
371 static const uptr kShadowMsk = 0x3c0000000000ull;
372 static const uptr kShadowXor = 0x020000000000ull;
373 static const uptr kShadowAdd = 0x000000000000ull;
374 static const uptr kVdsoBeg = 0x7800000000000000ull;
375 static const uptr kMidAppMemBeg = 0;
376 static const uptr kMidAppMemEnd = 0;
377};
378
379/*
380C/C++ on linux/powerpc64 (47-bit VMA)
3810000 0000 1000 - 0100 0000 0000: main binary
3820100 0000 0000 - 0200 0000 0000: -
3830100 0000 0000 - 0800 0000 0000: shadow
3840800 0000 0000 - 1000 0000 0000: -
3851000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3861200 0000 0000 - 7d00 0000 0000: -
3877d00 0000 0000 - 7e00 0000 0000: heap
3887e00 0000 0000 - 7e80 0000 0000: -
3897e80 0000 0000 - 8000 0000 0000: modules and main thread stack
390*/
391struct MappingPPC64_47 {
392 static const uptr kMetaShadowBeg = 0x100000000000ull;
393 static const uptr kMetaShadowEnd = 0x120000000000ull;
394 static const uptr kShadowBeg = 0x010000000000ull;
395 static const uptr kShadowEnd = 0x080000000000ull;
396 static const uptr kHeapMemBeg = 0x7d0000000000ull;
397 static const uptr kHeapMemEnd = 0x7e0000000000ull;
398 static const uptr kLoAppMemBeg = 0x000000001000ull;
399 static const uptr kLoAppMemEnd = 0x010000000000ull;
400 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
401 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
402 static const uptr kShadowMsk = 0x7c0000000000ull;
403 static const uptr kShadowXor = 0x020000000000ull;
404 static const uptr kShadowAdd = 0x000000000000ull;
405 static const uptr kVdsoBeg = 0x7800000000000000ull;
406 static const uptr kMidAppMemBeg = 0;
407 static const uptr kMidAppMemEnd = 0;
408};
409
410/*
411C/C++ on linux/riscv64 (39-bit VMA)
4120000 0010 00 - 0200 0000 00: main binary ( 8 GB)
4130200 0000 00 - 1000 0000 00: -
4141000 0000 00 - 4000 0000 00: shadow memory (64 GB)
4154000 0000 00 - 4800 0000 00: metainfo (16 GB)
4164800 0000 00 - 5500 0000 00: -
4175500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
4185600 0000 00 - 7c00 0000 00: -
4197d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
420
421mmap by default allocates from top downwards
422VDSO sits below loader and above dynamic libraries, within HiApp region.
423Heap starts after program region whose position depends on pie or non-pie.
424Disable tracking them since their locations are not fixed.
425*/
426struct MappingRiscv64_39 {
427 static const uptr kLoAppMemBeg = 0x0000001000ull;
428 static const uptr kLoAppMemEnd = 0x0200000000ull;
429 static const uptr kShadowBeg = 0x1000000000ull;
430 static const uptr kShadowEnd = 0x2000000000ull;
431 static const uptr kMetaShadowBeg = 0x2000000000ull;
432 static const uptr kMetaShadowEnd = 0x2400000000ull;
433 static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
434 static const uptr kMidAppMemEnd = 0x2c00000000ull;
435 static const uptr kHeapMemBeg = 0x2c00000000ull;
436 static const uptr kHeapMemEnd = 0x2c00000000ull;
437 static const uptr kHiAppMemBeg = 0x3c00000000ull;
438 static const uptr kHiAppMemEnd = 0x4000000000ull;
439 static const uptr kShadowMsk = 0x3800000000ull;
440 static const uptr kShadowXor = 0x0800000000ull;
441 static const uptr kShadowAdd = 0x0000000000ull;
442 static const uptr kVdsoBeg = 0x4000000000ull;
443};
444
445/*
446C/C++ on linux/riscv64 (48-bit VMA)
4470000 0000 1000 - 0400 0000 0000: main binary ( 4 TB)
4480500 0000 0000 - 2000 0000 0000: -
4492000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
4504000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
4514800 0000 0000 - 5555 5555 5000: -
4525555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
4535a00 0000 0000 - 7a00 0000 0000: -
4547a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB)
455*/
456struct MappingRiscv64_48 {
457 static const uptr kLoAppMemBeg = 0x000000001000ull;
458 static const uptr kLoAppMemEnd = 0x040000000000ull;
459 static const uptr kShadowBeg = 0x200000000000ull;
460 static const uptr kShadowEnd = 0x400000000000ull;
461 static const uptr kMetaShadowBeg = 0x400000000000ull;
462 static const uptr kMetaShadowEnd = 0x480000000000ull;
463 static const uptr kMidAppMemBeg = 0x555555555000ull;
464 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
465 static const uptr kHeapMemBeg = 0x5a0000000000ull;
466 static const uptr kHeapMemEnd = 0x5a0000000000ull;
467 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
468 static const uptr kHiAppMemEnd = 0x800000000000ull;
469 static const uptr kShadowMsk = 0x700000000000ull;
470 static const uptr kShadowXor = 0x100000000000ull;
471 static const uptr kShadowAdd = 0x000000000000ull;
472 static const uptr kVdsoBeg = 0x800000000000ull;
473};
474
475/*
476C/C++ on linux/s390x
477While the kernel provides a 64-bit address space, we have to restrict ourselves
478to 48 bits due to how e.g. SyncVar::GetId() works.
4790000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
4800e00 0000 0000 - 2000 0000 0000: -
4812000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
4824000 0000 0000 - 9000 0000 0000: -
4839000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
4849800 0000 0000 - be00 0000 0000: -
485be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
486*/
487struct MappingS390x {
488 static const uptr kMetaShadowBeg = 0x900000000000ull;
489 static const uptr kMetaShadowEnd = 0x980000000000ull;
490 static const uptr kShadowBeg = 0x200000000000ull;
491 static const uptr kShadowEnd = 0x400000000000ull;
492 static const uptr kHeapMemBeg = 0xbe0000000000ull;
493 static const uptr kHeapMemEnd = 0xc00000000000ull;
494 static const uptr kLoAppMemBeg = 0x000000001000ull;
495 static const uptr kLoAppMemEnd = 0x0e0000000000ull;
496 static const uptr kHiAppMemBeg = 0xc00000004000ull;
497 static const uptr kHiAppMemEnd = 0xc00000004000ull;
498 static const uptr kShadowMsk = 0xb00000000000ull;
499 static const uptr kShadowXor = 0x100000000000ull;
500 static const uptr kShadowAdd = 0x000000000000ull;
501 static const uptr kVdsoBeg = 0xfffffffff000ull;
502 static const uptr kMidAppMemBeg = 0;
503 static const uptr kMidAppMemEnd = 0;
504};
505
506/* Go on linux, darwin and freebsd on x86_64
5070000 0000 1000 - 0000 1000 0000: executable
5080000 1000 0000 - 00c0 0000 0000: -
50900c0 0000 0000 - 00e0 0000 0000: heap
51000e0 0000 0000 - 2000 0000 0000: -
5112000 0000 0000 - 21c0 0000 0000: shadow
51221c0 0000 0000 - 3000 0000 0000: -
5133000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
5144000 0000 0000 - 8000 0000 0000: -
515*/
516
517struct MappingGo48 {
518 static const uptr kMetaShadowBeg = 0x300000000000ull;
519 static const uptr kMetaShadowEnd = 0x400000000000ull;
520 static const uptr kShadowBeg = 0x200000000000ull;
521 static const uptr kShadowEnd = 0x21c000000000ull;
522 static const uptr kLoAppMemBeg = 0x000000001000ull;
523 static const uptr kLoAppMemEnd = 0x00e000000000ull;
524 static const uptr kMidAppMemBeg = 0;
525 static const uptr kMidAppMemEnd = 0;
526 static const uptr kHiAppMemBeg = 0;
527 static const uptr kHiAppMemEnd = 0;
528 static const uptr kHeapMemBeg = 0;
529 static const uptr kHeapMemEnd = 0;
530 static const uptr kVdsoBeg = 0;
531 static const uptr kShadowMsk = 0;
532 static const uptr kShadowXor = 0;
533 static const uptr kShadowAdd = 0x200000000000ull;
534};
535
536/* Go on windows
5370000 0000 1000 - 0000 1000 0000: executable
5380000 1000 0000 - 00f8 0000 0000: -
53900c0 0000 0000 - 00e0 0000 0000: heap
54000e0 0000 0000 - 0100 0000 0000: -
5410100 0000 0000 - 0300 0000 0000: shadow
5420300 0000 0000 - 0700 0000 0000: -
5430700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
54407d0 0000 0000 - 8000 0000 0000: -
545PIE binaries currently not supported, but it should be theoretically possible.
546*/
547
548struct MappingGoWindows {
549 static const uptr kMetaShadowBeg = 0x070000000000ull;
550 static const uptr kMetaShadowEnd = 0x077000000000ull;
551 static const uptr kShadowBeg = 0x010000000000ull;
552 static const uptr kShadowEnd = 0x030000000000ull;
553 static const uptr kLoAppMemBeg = 0x000000001000ull;
554 static const uptr kLoAppMemEnd = 0x00e000000000ull;
555 static const uptr kMidAppMemBeg = 0;
556 static const uptr kMidAppMemEnd = 0;
557 static const uptr kHiAppMemBeg = 0;
558 static const uptr kHiAppMemEnd = 0;
559 static const uptr kHeapMemBeg = 0;
560 static const uptr kHeapMemEnd = 0;
561 static const uptr kVdsoBeg = 0;
562 static const uptr kShadowMsk = 0;
563 static const uptr kShadowXor = 0;
564 static const uptr kShadowAdd = 0x010000000000ull;
565};
566
567/* Go on linux/powerpc64 (46-bit VMA)
5680000 0000 1000 - 0000 1000 0000: executable
5690000 1000 0000 - 00c0 0000 0000: -
57000c0 0000 0000 - 00e0 0000 0000: heap
57100e0 0000 0000 - 2000 0000 0000: -
5722000 0000 0000 - 21c0 0000 0000: shadow
57321c0 0000 0000 - 2400 0000 0000: -
5742400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
5752470 0000 0000 - 4000 0000 0000: -
576*/
577
578struct MappingGoPPC64_46 {
579 static const uptr kMetaShadowBeg = 0x240000000000ull;
580 static const uptr kMetaShadowEnd = 0x247000000000ull;
581 static const uptr kShadowBeg = 0x200000000000ull;
582 static const uptr kShadowEnd = 0x21c000000000ull;
583 static const uptr kLoAppMemBeg = 0x000000001000ull;
584 static const uptr kLoAppMemEnd = 0x00e000000000ull;
585 static const uptr kMidAppMemBeg = 0;
586 static const uptr kMidAppMemEnd = 0;
587 static const uptr kHiAppMemBeg = 0;
588 static const uptr kHiAppMemEnd = 0;
589 static const uptr kHeapMemBeg = 0;
590 static const uptr kHeapMemEnd = 0;
591 static const uptr kVdsoBeg = 0;
592 static const uptr kShadowMsk = 0;
593 static const uptr kShadowXor = 0;
594 static const uptr kShadowAdd = 0x200000000000ull;
595};
596
597/* Go on linux/powerpc64 (47-bit VMA)
5980000 0000 1000 - 0000 1000 0000: executable
5990000 1000 0000 - 00c0 0000 0000: -
60000c0 0000 0000 - 00e0 0000 0000: heap
60100e0 0000 0000 - 2000 0000 0000: -
6022000 0000 0000 - 2800 0000 0000: shadow
6032800 0000 0000 - 3000 0000 0000: -
6043000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6053200 0000 0000 - 8000 0000 0000: -
606*/
607
608struct MappingGoPPC64_47 {
609 static const uptr kMetaShadowBeg = 0x300000000000ull;
610 static const uptr kMetaShadowEnd = 0x320000000000ull;
611 static const uptr kShadowBeg = 0x200000000000ull;
612 static const uptr kShadowEnd = 0x280000000000ull;
613 static const uptr kLoAppMemBeg = 0x000000001000ull;
614 static const uptr kLoAppMemEnd = 0x00e000000000ull;
615 static const uptr kMidAppMemBeg = 0;
616 static const uptr kMidAppMemEnd = 0;
617 static const uptr kHiAppMemBeg = 0;
618 static const uptr kHiAppMemEnd = 0;
619 static const uptr kHeapMemBeg = 0;
620 static const uptr kHeapMemEnd = 0;
621 static const uptr kVdsoBeg = 0;
622 static const uptr kShadowMsk = 0;
623 static const uptr kShadowXor = 0;
624 static const uptr kShadowAdd = 0x200000000000ull;
625};
626
627/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
6280000 0000 1000 - 0000 1000 0000: executable
6290000 1000 0000 - 00c0 0000 0000: -
63000c0 0000 0000 - 00e0 0000 0000: heap
63100e0 0000 0000 - 2000 0000 0000: -
6322000 0000 0000 - 2800 0000 0000: shadow
6332800 0000 0000 - 3000 0000 0000: -
6343000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6353200 0000 0000 - 8000 0000 0000: -
636*/
637struct MappingGoAarch64 {
638 static const uptr kMetaShadowBeg = 0x300000000000ull;
639 static const uptr kMetaShadowEnd = 0x320000000000ull;
640 static const uptr kShadowBeg = 0x200000000000ull;
641 static const uptr kShadowEnd = 0x280000000000ull;
642 static const uptr kLoAppMemBeg = 0x000000001000ull;
643 static const uptr kLoAppMemEnd = 0x00e000000000ull;
644 static const uptr kMidAppMemBeg = 0;
645 static const uptr kMidAppMemEnd = 0;
646 static const uptr kHiAppMemBeg = 0;
647 static const uptr kHiAppMemEnd = 0;
648 static const uptr kHeapMemBeg = 0;
649 static const uptr kHeapMemEnd = 0;
650 static const uptr kVdsoBeg = 0;
651 static const uptr kShadowMsk = 0;
652 static const uptr kShadowXor = 0;
653 static const uptr kShadowAdd = 0x200000000000ull;
654};
655
656/* Go on linux/loongarch64 (47-bit VMA)
6570000 0000 1000 - 0000 1000 0000: executable
6580000 1000 0000 - 00c0 0000 0000: -
65900c0 0000 0000 - 00e0 0000 0000: heap
66000e0 0000 0000 - 2000 0000 0000: -
6612000 0000 0000 - 2800 0000 0000: shadow
6622800 0000 0000 - 3000 0000 0000: -
6633000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6643200 0000 0000 - 8000 0000 0000: -
665*/
666struct MappingGoLoongArch64_47 {
667 static const uptr kMetaShadowBeg = 0x300000000000ull;
668 static const uptr kMetaShadowEnd = 0x320000000000ull;
669 static const uptr kShadowBeg = 0x200000000000ull;
670 static const uptr kShadowEnd = 0x280000000000ull;
671 static const uptr kLoAppMemBeg = 0x000000001000ull;
672 static const uptr kLoAppMemEnd = 0x00e000000000ull;
673 static const uptr kMidAppMemBeg = 0;
674 static const uptr kMidAppMemEnd = 0;
675 static const uptr kHiAppMemBeg = 0;
676 static const uptr kHiAppMemEnd = 0;
677 static const uptr kHeapMemBeg = 0;
678 static const uptr kHeapMemEnd = 0;
679 static const uptr kVdsoBeg = 0;
680 static const uptr kShadowMsk = 0;
681 static const uptr kShadowXor = 0;
682 static const uptr kShadowAdd = 0x200000000000ull;
683};
684
685/*
686Go on linux/mips64 (47-bit VMA)
6870000 0000 1000 - 0000 1000 0000: executable
6880000 1000 0000 - 00c0 0000 0000: -
68900c0 0000 0000 - 00e0 0000 0000: heap
69000e0 0000 0000 - 2000 0000 0000: -
6912000 0000 0000 - 2800 0000 0000: shadow
6922800 0000 0000 - 3000 0000 0000: -
6933000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6943200 0000 0000 - 8000 0000 0000: -
695*/
696struct MappingGoMips64_47 {
697 static const uptr kMetaShadowBeg = 0x300000000000ull;
698 static const uptr kMetaShadowEnd = 0x320000000000ull;
699 static const uptr kShadowBeg = 0x200000000000ull;
700 static const uptr kShadowEnd = 0x280000000000ull;
701 static const uptr kLoAppMemBeg = 0x000000001000ull;
702 static const uptr kLoAppMemEnd = 0x00e000000000ull;
703 static const uptr kMidAppMemBeg = 0;
704 static const uptr kMidAppMemEnd = 0;
705 static const uptr kHiAppMemBeg = 0;
706 static const uptr kHiAppMemEnd = 0;
707 static const uptr kHeapMemBeg = 0;
708 static const uptr kHeapMemEnd = 0;
709 static const uptr kVdsoBeg = 0;
710 static const uptr kShadowMsk = 0;
711 static const uptr kShadowXor = 0;
712 static const uptr kShadowAdd = 0x200000000000ull;
713};
714
715/* Go on linux/riscv64 (39-bit VMA)
7160000 0001 0000 - 000f 0000 0000: executable and heap (60 GiB)
717000f 0000 0000 - 0010 0000 0000: -
7180010 0000 0000 - 0030 0000 0000: shadow - 128 GiB ( ~ 2 * app)
7190030 0000 0000 - 0038 0000 0000: metainfo - 32 GiB ( ~ 0.5 * app)
7200038 0000 0000 - 0040 0000 0000: -
721*/
722struct MappingGoRiscv64_39 {
723 static const uptr kMetaShadowBeg = 0x003000000000ull;
724 static const uptr kMetaShadowEnd = 0x003800000000ull;
725 static const uptr kShadowBeg = 0x001000000000ull;
726 static const uptr kShadowEnd = 0x003000000000ull;
727 static const uptr kLoAppMemBeg = 0x000000010000ull;
728 static const uptr kLoAppMemEnd = 0x000f00000000ull;
729 static const uptr kMidAppMemBeg = 0;
730 static const uptr kMidAppMemEnd = 0;
731 static const uptr kHiAppMemBeg = 0;
732 static const uptr kHiAppMemEnd = 0;
733 static const uptr kHeapMemBeg = 0;
734 static const uptr kHeapMemEnd = 0;
735 static const uptr kVdsoBeg = 0;
736 static const uptr kShadowMsk = 0;
737 static const uptr kShadowXor = 0;
738 static const uptr kShadowAdd = 0x001000000000ull;
739};
740
741/* Go on linux/riscv64 (48-bit VMA)
7420000 0001 0000 - 00e0 0000 0000: executable and heap (896 GiB)
74300e0 0000 0000 - 2000 0000 0000: -
7442000 0000 0000 - 2400 0000 0000: shadow - 4 TiB ( ~ 4 * app)
7452400 0000 0000 - 3000 0000 0000: -
7463000 0000 0000 - 3100 0000 0000: metainfo - 1 TiB ( ~ 1 * app)
7473100 0000 0000 - 8000 0000 0000: -
748*/
749struct MappingGoRiscv64_48 {
750 static const uptr kMetaShadowBeg = 0x300000000000ull;
751 static const uptr kMetaShadowEnd = 0x310000000000ull;
752 static const uptr kShadowBeg = 0x200000000000ull;
753 static const uptr kShadowEnd = 0x240000000000ull;
754 static const uptr kLoAppMemBeg = 0x000000010000ull;
755 static const uptr kLoAppMemEnd = 0x00e000000000ull;
756 static const uptr kMidAppMemBeg = 0;
757 static const uptr kMidAppMemEnd = 0;
758 static const uptr kHiAppMemBeg = 0;
759 static const uptr kHiAppMemEnd = 0;
760 static const uptr kHeapMemBeg = 0;
761 static const uptr kHeapMemEnd = 0;
762 static const uptr kVdsoBeg = 0;
763 static const uptr kShadowMsk = 0;
764 static const uptr kShadowXor = 0;
765 static const uptr kShadowAdd = 0x200000000000ull;
766};
767
768/*
769Go on linux/s390x
7700000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
7711000 0000 0000 - 4000 0000 0000: -
7724000 0000 0000 - 6000 0000 0000: shadow - 32 TiB (2 * app)
7736000 0000 0000 - 7000 0000 0000: -
7747000 0000 0000 - 7800 0000 0000: metainfo - 8 TiB (0.5 * app)
7757800 0000 0000 - 8000 0000 0000: -
776*/
777struct MappingGoS390x {
778 // Keep the mapping below 2^47 for QEMU linux-user on x86-64 hosts with
779 // four-level page tables.
780 static const uptr kMetaShadowBeg = 0x700000000000ull;
781 static const uptr kMetaShadowEnd = 0x780000000000ull;
782 static const uptr kShadowBeg = 0x400000000000ull;
783 static const uptr kShadowEnd = 0x600000000000ull;
784 static const uptr kLoAppMemBeg = 0x000000001000ull;
785 static const uptr kLoAppMemEnd = 0x100000000000ull;
786 static const uptr kMidAppMemBeg = 0;
787 static const uptr kMidAppMemEnd = 0;
788 static const uptr kHiAppMemBeg = 0;
789 static const uptr kHiAppMemEnd = 0;
790 static const uptr kHeapMemBeg = 0;
791 static const uptr kHeapMemEnd = 0;
792 static const uptr kVdsoBeg = 0;
793 static const uptr kShadowMsk = 0;
794 static const uptr kShadowXor = 0;
795 static const uptr kShadowAdd = 0x400000000000ull;
796};
797
798extern uptr vmaSize;
799
800template <typename Func, typename Arg>
801ALWAYS_INLINE auto SelectMapping(Arg arg) {
802#if SANITIZER_GO
803# if defined(__powerpc64__)
804 switch (vmaSize) {
805 case 46:
806 return Func::template Apply<MappingGoPPC64_46>(arg);
807 case 47:
808 return Func::template Apply<MappingGoPPC64_47>(arg);
809 }
810# elif defined(__mips64)
811 return Func::template Apply<MappingGoMips64_47>(arg);
812# elif defined(__s390x__)
813 return Func::template Apply<MappingGoS390x>(arg);
814# elif defined(__aarch64__)
815 return Func::template Apply<MappingGoAarch64>(arg);
816# elif defined(__loongarch_lp64)
817 return Func::template Apply<MappingGoLoongArch64_47>(arg);
818# elif SANITIZER_RISCV64
819 switch (vmaSize) {
820 case 39:
821 return Func::template Apply<MappingGoRiscv64_39>(arg);
822 case 48:
823 return Func::template Apply<MappingGoRiscv64_48>(arg);
824 }
825# elif SANITIZER_WINDOWS
826 return Func::template Apply<MappingGoWindows>(arg);
827# else
828 return Func::template Apply<MappingGo48>(arg);
829# endif
830#else // SANITIZER_GO
831# if SANITIZER_IOS && !SANITIZER_IOSSIM
832 return Func::template Apply<MappingAppleAarch64>(arg);
833# elif defined(__x86_64__) || SANITIZER_APPLE
834 return Func::template Apply<Mapping48AddressSpace>(arg);
835# elif defined(__aarch64__)
836 switch (vmaSize) {
837 case 39:
838 return Func::template Apply<MappingAarch64_39>(arg);
839 case 42:
840 return Func::template Apply<MappingAarch64_42>(arg);
841 case 47:
842 return Func::template Apply<MappingAarch64_47>(arg);
843 case 48:
844 return Func::template Apply<MappingAarch64_48>(arg);
845 }
846# elif SANITIZER_LOONGARCH64
847 return Func::template Apply<MappingLoongArch64_47>(arg);
848# elif defined(__powerpc64__)
849 switch (vmaSize) {
850 case 44:
851 return Func::template Apply<MappingPPC64_44>(arg);
852 case 46:
853 return Func::template Apply<MappingPPC64_46>(arg);
854 case 47:
855 return Func::template Apply<MappingPPC64_47>(arg);
856 }
857# elif defined(__mips64)
858 return Func::template Apply<MappingMips64_40>(arg);
859# elif SANITIZER_RISCV64
860 switch (vmaSize) {
861 case 39:
862 return Func::template Apply<MappingRiscv64_39>(arg);
863 case 48:
864 return Func::template Apply<MappingRiscv64_48>(arg);
865 }
866# elif defined(__s390x__)
867 return Func::template Apply<MappingS390x>(arg);
868# else
869# error "unsupported platform"
870# endif
871#endif
872 Die();
873}
874
875template <typename Func>
876void ForEachMapping() {
877 Func::template Apply<Mapping48AddressSpace>();
878 Func::template Apply<MappingMips64_40>();
879 Func::template Apply<MappingAppleAarch64>();
880 Func::template Apply<MappingAarch64_39>();
881 Func::template Apply<MappingAarch64_42>();
882 Func::template Apply<MappingAarch64_47>();
883 Func::template Apply<MappingAarch64_48>();
884 Func::template Apply<MappingLoongArch64_47>();
885 Func::template Apply<MappingPPC64_44>();
886 Func::template Apply<MappingPPC64_46>();
887 Func::template Apply<MappingPPC64_47>();
888 Func::template Apply<MappingRiscv64_39>();
889 Func::template Apply<MappingRiscv64_48>();
890 Func::template Apply<MappingS390x>();
891 Func::template Apply<MappingGo48>();
892 Func::template Apply<MappingGoWindows>();
893 Func::template Apply<MappingGoPPC64_46>();
894 Func::template Apply<MappingGoPPC64_47>();
895 Func::template Apply<MappingGoAarch64>();
896 Func::template Apply<MappingGoLoongArch64_47>();
897 Func::template Apply<MappingGoMips64_47>();
898 Func::template Apply<MappingGoRiscv64_39>();
899 Func::template Apply<MappingGoRiscv64_48>();
900 Func::template Apply<MappingGoS390x>();
901}
902
903enum MappingType {
904 kLoAppMemBeg,
905 kLoAppMemEnd,
906 kHiAppMemBeg,
907 kHiAppMemEnd,
908 kMidAppMemBeg,
909 kMidAppMemEnd,
910 kHeapMemBeg,
911 kHeapMemEnd,
912 kShadowBeg,
913 kShadowEnd,
914 kMetaShadowBeg,
915 kMetaShadowEnd,
916 kVdsoBeg,
917};
918
919struct MappingField {
920 template <typename Mapping>
921 static uptr Apply(MappingType type) {
922 switch (type) {
923 case kLoAppMemBeg:
924 return Mapping::kLoAppMemBeg;
925 case kLoAppMemEnd:
926 return Mapping::kLoAppMemEnd;
927 case kMidAppMemBeg:
928 return Mapping::kMidAppMemBeg;
929 case kMidAppMemEnd:
930 return Mapping::kMidAppMemEnd;
931 case kHiAppMemBeg:
932 return Mapping::kHiAppMemBeg;
933 case kHiAppMemEnd:
934 return Mapping::kHiAppMemEnd;
935 case kHeapMemBeg:
936 return Mapping::kHeapMemBeg;
937 case kHeapMemEnd:
938 return Mapping::kHeapMemEnd;
939 case kVdsoBeg:
940 return Mapping::kVdsoBeg;
941 case kShadowBeg:
942 return Mapping::kShadowBeg;
943 case kShadowEnd:
944 return Mapping::kShadowEnd;
945 case kMetaShadowBeg:
946 return Mapping::kMetaShadowBeg;
947 case kMetaShadowEnd:
948 return Mapping::kMetaShadowEnd;
949 }
950 Die();
951 }
952};
953
954ALWAYS_INLINE
955uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(arg: kLoAppMemBeg); }
956ALWAYS_INLINE
957uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(arg: kLoAppMemEnd); }
958
959ALWAYS_INLINE
960uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(arg: kMidAppMemBeg); }
961ALWAYS_INLINE
962uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(arg: kMidAppMemEnd); }
963
964ALWAYS_INLINE
965uptr HeapMemBeg(void) { return SelectMapping<MappingField>(arg: kHeapMemBeg); }
966ALWAYS_INLINE
967uptr HeapMemEnd(void) { return SelectMapping<MappingField>(arg: kHeapMemEnd); }
968
969ALWAYS_INLINE
970uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(arg: kHiAppMemBeg); }
971ALWAYS_INLINE
972uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(arg: kHiAppMemEnd); }
973
974ALWAYS_INLINE
975uptr VdsoBeg(void) { return SelectMapping<MappingField>(arg: kVdsoBeg); }
976
977ALWAYS_INLINE
978uptr ShadowBeg(void) { return SelectMapping<MappingField>(arg: kShadowBeg); }
979ALWAYS_INLINE
980uptr ShadowEnd(void) { return SelectMapping<MappingField>(arg: kShadowEnd); }
981
982ALWAYS_INLINE
983uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(arg: kMetaShadowBeg); }
984ALWAYS_INLINE
985uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(arg: kMetaShadowEnd); }
986
987struct IsAppMemImpl {
988 template <typename Mapping>
989 static bool Apply(uptr mem) {
990 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
991 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
992 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
993 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
994 }
995};
996
997ALWAYS_INLINE
998bool IsAppMem(uptr mem) {
999 return SelectMapping<IsAppMemImpl>(STRIP_MTE_TAG(mem));
1000}
1001
1002struct IsShadowMemImpl {
1003 template <typename Mapping>
1004 static bool Apply(uptr mem) {
1005 return mem >= Mapping::kShadowBeg && mem < Mapping::kShadowEnd;
1006 }
1007};
1008
1009ALWAYS_INLINE
1010bool IsShadowMem(RawShadow *p) {
1011 return SelectMapping<IsShadowMemImpl>(arg: reinterpret_cast<uptr>(p));
1012}
1013
1014struct IsMetaMemImpl {
1015 template <typename Mapping>
1016 static bool Apply(uptr mem) {
1017 return mem >= Mapping::kMetaShadowBeg && mem < Mapping::kMetaShadowEnd;
1018 }
1019};
1020
1021ALWAYS_INLINE
1022bool IsMetaMem(const u32 *p) {
1023 return SelectMapping<IsMetaMemImpl>(arg: reinterpret_cast<uptr>(p));
1024}
1025
1026struct MemToShadowImpl {
1027 template <typename Mapping>
1028 static uptr Apply(uptr x) {
1029 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
1030 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
1031 Mapping::kShadowXor) *
1032 kShadowMultiplier +
1033 Mapping::kShadowAdd;
1034 }
1035};
1036
1037ALWAYS_INLINE
1038RawShadow *MemToShadow(uptr x) {
1039 return reinterpret_cast<RawShadow*>(
1040 SelectMapping<MemToShadowImpl>(STRIP_MTE_TAG(x)));
1041}
1042
1043struct MemToMetaImpl {
1044 template <typename Mapping>
1045 static u32 *Apply(uptr x) {
1046 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
1047 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
1048 kMetaShadowCell * kMetaShadowSize) |
1049 Mapping::kMetaShadowBeg);
1050 }
1051};
1052
1053ALWAYS_INLINE
1054u32* MemToMeta(uptr x) {
1055 return SelectMapping<MemToMetaImpl>(STRIP_MTE_TAG(x));
1056}
1057
1058struct ShadowToMemImpl {
1059 template <typename Mapping>
1060 static uptr Apply(uptr sp) {
1061 if (!IsShadowMemImpl::Apply<Mapping>(sp))
1062 return 0;
1063 // The shadow mapping is non-linear and we've lost some bits, so we don't
1064 // have an easy way to restore the original app address. But the mapping is
1065 // a bijection, so we try to restore the address as belonging to
1066 // low/mid/high range consecutively and see if shadow->app->shadow mapping
1067 // gives us the same address.
1068 uptr p =
1069 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
1070 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
1071 MemToShadowImpl::Apply<Mapping>(p) == sp)
1072 return p;
1073 if (Mapping::kMidAppMemBeg) {
1074 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
1075 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
1076 MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
1077 return p_mid;
1078 }
1079 return p | Mapping::kShadowMsk;
1080 }
1081};
1082
1083ALWAYS_INLINE
1084uptr ShadowToMem(RawShadow *s) {
1085 return SelectMapping<ShadowToMemImpl>(arg: reinterpret_cast<uptr>(s));
1086}
1087
1088// Compresses addr to kCompressedAddrBits stored in least significant bits.
1089ALWAYS_INLINE uptr CompressAddr(uptr addr) {
1090 return addr & ((1ull << kCompressedAddrBits) - 1);
1091}
1092
1093struct RestoreAddrImpl {
1094 typedef uptr Result;
1095 template <typename Mapping>
1096 static Result Apply(uptr addr) {
1097 // To restore the address we go over all app memory ranges and check if top
1098 // 3 bits of the compressed addr match that of the app range. If yes, we
1099 // assume that the compressed address come from that range and restore the
1100 // missing top bits to match the app range address.
1101 const uptr ranges[] = {
1102 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
1103 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
1104 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
1105 };
1106 const uptr indicator = 0x0e0000000000ull;
1107 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(x: indicator);
1108 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
1109 uptr beg = ranges[i];
1110 uptr end = ranges[i + 1];
1111 if (beg == end)
1112 continue;
1113 for (uptr p = beg; p < end; p = RoundDown(p: p + ind_lsb, align: ind_lsb)) {
1114 if ((addr & indicator) == (p & indicator))
1115 return addr | (p & ~(ind_lsb - 1));
1116 }
1117 }
1118 Printf(format: "ThreadSanitizer: failed to restore address 0x%zx\n", addr);
1119 Die();
1120 }
1121};
1122
1123// Restores compressed addr from kCompressedAddrBits to full representation.
1124// This is called only during reporting and is not performance-critical.
1125inline uptr RestoreAddr(uptr addr) {
1126 return SelectMapping<RestoreAddrImpl>(arg: addr);
1127}
1128
1129void InitializePlatform();
1130void InitializePlatformEarly();
1131bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
1132void InitializeShadowMemoryPlatform();
1133void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1134int ExtractResolvFDs(void *state, int *fds, int nfd);
1135int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
1136uptr ExtractLongJmpSp(uptr *env);
1137void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
1138
1139int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
1140 void (*cleanup)(void *arg), void *arg);
1141
1142void DestroyThreadState();
1143void PlatformCleanUpThreadState(ThreadState *thr);
1144
1145} // namespace __tsan
1146
1147#endif // TSAN_PLATFORM_H
1148