1//===-- dfsan.cpp ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of DataFlowSanitizer.
10//
11// DataFlowSanitizer runtime. This file defines the public interface to
12// DataFlowSanitizer as well as the definition of certain runtime functions
13// called automatically by the compiler (specifically the instrumentation pass
14// in llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp).
15//
16// The public interface is defined in include/sanitizer/dfsan_interface.h whose
17// functions are prefixed dfsan_ while the compiler interface functions are
18// prefixed __dfsan_.
19//===----------------------------------------------------------------------===//
20
21#include "dfsan/dfsan.h"
22
23#include "dfsan/dfsan_chained_origin_depot.h"
24#include "dfsan/dfsan_flags.h"
25#include "dfsan/dfsan_origin.h"
26#include "dfsan/dfsan_thread.h"
27#include "sanitizer_common/sanitizer_atomic.h"
28#include "sanitizer_common/sanitizer_common.h"
29#include "sanitizer_common/sanitizer_file.h"
30#include "sanitizer_common/sanitizer_flag_parser.h"
31#include "sanitizer_common/sanitizer_flags.h"
32#include "sanitizer_common/sanitizer_internal_defs.h"
33#include "sanitizer_common/sanitizer_libc.h"
34#include "sanitizer_common/sanitizer_report_decorator.h"
35#include "sanitizer_common/sanitizer_stacktrace.h"
36#if SANITIZER_LINUX
37# include <sys/personality.h>
38#endif
39
40using namespace __dfsan;
41
42Flags __dfsan::flags_data;
43
44// The size of TLS variables. These constants must be kept in sync with the ones
45// in DataFlowSanitizer.cpp.
46static const int kDFsanArgTlsSize = 800;
47static const int kDFsanRetvalTlsSize = 800;
48static const int kDFsanArgOriginTlsSize = 800;
49
50SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
51 __dfsan_retval_tls[kDFsanRetvalTlsSize / sizeof(u64)];
52SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32 __dfsan_retval_origin_tls;
53SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
54 __dfsan_arg_tls[kDFsanArgTlsSize / sizeof(u64)];
55SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32
56 __dfsan_arg_origin_tls[kDFsanArgOriginTlsSize / sizeof(u32)];
57
58// Instrumented code may set this value in terms of -dfsan-track-origins.
59// * undefined or 0: do not track origins.
60// * 1: track origins at memory store operations.
61// * 2: track origins at memory load and store operations.
62// TODO: track callsites.
63extern "C" SANITIZER_WEAK_ATTRIBUTE const int __dfsan_track_origins;
64
65extern "C" SANITIZER_INTERFACE_ATTRIBUTE int dfsan_get_track_origins() {
66 return &__dfsan_track_origins ? __dfsan_track_origins : 0;
67}
68
69// On Linux/x86_64, memory is laid out as follows:
70//
71// +--------------------+ 0x800000000000 (top of memory)
72// | application 3 |
73// +--------------------+ 0x700000000000
74// | invalid |
75// +--------------------+ 0x610000000000
76// | origin 1 |
77// +--------------------+ 0x600000000000
78// | application 2 |
79// +--------------------+ 0x510000000000
80// | shadow 1 |
81// +--------------------+ 0x500000000000
82// | invalid |
83// +--------------------+ 0x400000000000
84// | origin 3 |
85// +--------------------+ 0x300000000000
86// | shadow 3 |
87// +--------------------+ 0x200000000000
88// | origin 2 |
89// +--------------------+ 0x110000000000
90// | invalid |
91// +--------------------+ 0x100000000000
92// | shadow 2 |
93// +--------------------+ 0x010000000000
94// | application 1 |
95// +--------------------+ 0x000000000000
96//
97// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
98// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
99
100extern "C" SANITIZER_INTERFACE_ATTRIBUTE
101dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
102 dfsan_label label = ls[0];
103 for (uptr i = 1; i != n; ++i)
104 label |= ls[i];
105 return label;
106}
107
108// Return the union of all the n labels from addr at the high 32 bit, and the
109// origin of the first taint byte at the low 32 bit.
110extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
111__dfsan_load_label_and_origin(const void *addr, uptr n) {
112 dfsan_label label = 0;
113 u64 ret = 0;
114 uptr p = (uptr)addr;
115 dfsan_label *s = shadow_for(ptr: (void *)p);
116 for (uptr i = 0; i < n; ++i) {
117 dfsan_label l = s[i];
118 if (!l)
119 continue;
120 label |= l;
121 if (!ret)
122 ret = *(dfsan_origin *)origin_for(ptr: (void *)(p + i));
123 }
124 return ret | (u64)label << 32;
125}
126
127extern "C" SANITIZER_INTERFACE_ATTRIBUTE
128void __dfsan_unimplemented(char *fname) {
129 if (flags().warn_unimplemented)
130 Report(format: "WARNING: DataFlowSanitizer: call to uninstrumented function %s\n",
131 fname);
132}
133
134extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_wrapper_extern_weak_null(
135 const void *addr, char *fname) {
136 if (!addr)
137 Report(
138 format: "ERROR: DataFlowSanitizer: dfsan generated wrapper calling null "
139 "extern_weak function %s\nIf this only happens with dfsan, the "
140 "dfsan instrumentation pass may be accidentally optimizing out a "
141 "null check\n",
142 fname);
143}
144
145// Use '-mllvm -dfsan-debug-nonzero-labels' and break on this function
146// to try to figure out where labels are being introduced in a nominally
147// label-free program.
148extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_nonzero_label() {
149 if (flags().warn_nonzero_labels)
150 Report(format: "WARNING: DataFlowSanitizer: saw nonzero label\n");
151}
152
153// Indirect call to an uninstrumented vararg function. We don't have a way of
154// handling these at the moment.
155extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
156__dfsan_vararg_wrapper(const char *fname) {
157 Report(format: "FATAL: DataFlowSanitizer: unsupported indirect call to vararg "
158 "function %s\n", fname);
159 Die();
160}
161
162// Resolves the union of two labels.
163SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
164dfsan_union(dfsan_label l1, dfsan_label l2) {
165 return l1 | l2;
166}
167
168static const uptr kOriginAlign = sizeof(dfsan_origin);
169static const uptr kOriginAlignMask = ~(kOriginAlign - 1UL);
170
171static uptr OriginAlignUp(uptr u) {
172 return (u + kOriginAlign - 1) & kOriginAlignMask;
173}
174
175static uptr OriginAlignDown(uptr u) { return u & kOriginAlignMask; }
176
177// Return the origin of the first taint byte in the size bytes from the address
178// addr.
179static dfsan_origin GetOriginIfTainted(uptr addr, uptr size) {
180 for (uptr i = 0; i < size; ++i, ++addr) {
181 dfsan_label *s = shadow_for(ptr: (void *)addr);
182
183 if (*s) {
184 // Validate address region.
185 CHECK(MEM_IS_SHADOW(s));
186 return *(dfsan_origin *)origin_for(ptr: (void *)addr);
187 }
188 }
189 return 0;
190}
191
192// For platforms which support slow unwinder only, we need to restrict the store
193// context size to 1, basically only storing the current pc, because the slow
194// unwinder which is based on libunwind is not async signal safe and causes
195// random freezes in forking applications as well as in signal handlers.
196// DFSan supports only Linux. So we do not restrict the store context size.
197#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
198 UNINITIALIZED BufferedStackTrace stack; \
199 stack.Unwind(pc, bp, nullptr, true, flags().store_context_size);
200
201#define PRINT_CALLER_STACK_TRACE \
202 { \
203 GET_CALLER_PC_BP; \
204 GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
205 stack.Print(); \
206 }
207
208// Return a chain with the previous ID id and the current stack.
209// from_init = true if this is the first chain of an origin tracking path.
210static u32 ChainOrigin(u32 id, StackTrace *stack, bool from_init = false) {
211 // StackDepot is not async signal safe. Do not create new chains in a signal
212 // handler.
213 DFsanThread *t = GetCurrentThread();
214 if (t && t->InSignalHandler())
215 return id;
216
217 // As an optimization the origin of an application byte is updated only when
218 // its shadow is non-zero. Because we are only interested in the origins of
219 // taint labels, it does not matter what origin a zero label has. This reduces
220 // memory write cost. MSan does similar optimization. The following invariant
221 // may not hold because of some bugs. We check the invariant to help debug.
222 if (!from_init && id == 0 && flags().check_origin_invariant) {
223 Printf(format: " DFSan found invalid origin invariant\n");
224 PRINT_CALLER_STACK_TRACE
225 }
226
227 Origin o = Origin::FromRawId(id);
228 stack->tag = StackTrace::TAG_UNKNOWN;
229 Origin chained = Origin::CreateChainedOrigin(prev: o, stack);
230 return chained.raw_id();
231}
232
233static void ChainAndWriteOriginIfTainted(uptr src, uptr size, uptr dst,
234 StackTrace *stack) {
235 dfsan_origin o = GetOriginIfTainted(addr: src, size);
236 if (o) {
237 o = ChainOrigin(id: o, stack);
238 *(dfsan_origin *)origin_for(ptr: (void *)dst) = o;
239 }
240}
241
242// Copy the origins of the size bytes from src to dst. The source and target
243// memory ranges cannot be overlapped. This is used by memcpy. stack records the
244// stack trace of the memcpy. When dst and src are not 4-byte aligned properly,
245// origins at the unaligned address boundaries may be overwritten because four
246// contiguous bytes share the same origin.
247static void CopyOrigin(const void *dst, const void *src, uptr size,
248 StackTrace *stack) {
249 uptr d = (uptr)dst;
250 uptr beg = OriginAlignDown(u: d);
251 // Copy left unaligned origin if that memory is tainted.
252 if (beg < d) {
253 ChainAndWriteOriginIfTainted(src: (uptr)src, size: beg + kOriginAlign - d, dst: beg, stack);
254 beg += kOriginAlign;
255 }
256
257 uptr end = OriginAlignDown(u: d + size);
258 // If both ends fall into the same 4-byte slot, we are done.
259 if (end < beg)
260 return;
261
262 // Copy right unaligned origin if that memory is tainted.
263 if (end < d + size)
264 ChainAndWriteOriginIfTainted(src: (uptr)src + (end - d), size: (d + size) - end, dst: end,
265 stack);
266
267 if (beg >= end)
268 return;
269
270 // Align src up.
271 uptr src_a = OriginAlignUp(u: (uptr)src);
272 dfsan_origin *src_o = origin_for(ptr: (void *)src_a);
273 u32 *src_s = (u32 *)shadow_for(ptr: (void *)src_a);
274 dfsan_origin *src_end = origin_for(ptr: (void *)(src_a + (end - beg)));
275 dfsan_origin *dst_o = origin_for(ptr: (void *)beg);
276 dfsan_origin last_src_o = 0;
277 dfsan_origin last_dst_o = 0;
278 for (; src_o < src_end; ++src_o, ++src_s, ++dst_o) {
279 if (!*src_s)
280 continue;
281 if (*src_o != last_src_o) {
282 last_src_o = *src_o;
283 last_dst_o = ChainOrigin(id: last_src_o, stack);
284 }
285 *dst_o = last_dst_o;
286 }
287}
288
289// Copy the origins of the size bytes from src to dst. The source and target
290// memory ranges may be overlapped. So the copy is done in a reverse order.
291// This is used by memmove. stack records the stack trace of the memmove.
292static void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
293 StackTrace *stack) {
294 uptr d = (uptr)dst;
295 uptr end = OriginAlignDown(u: d + size);
296
297 // Copy right unaligned origin if that memory is tainted.
298 if (end < d + size)
299 ChainAndWriteOriginIfTainted(src: (uptr)src + (end - d), size: (d + size) - end, dst: end,
300 stack);
301
302 uptr beg = OriginAlignDown(u: d);
303
304 if (beg + kOriginAlign < end) {
305 // Align src up.
306 uptr src_a = OriginAlignUp(u: (uptr)src);
307 void *src_end = (void *)(src_a + end - beg - kOriginAlign);
308 dfsan_origin *src_end_o = origin_for(ptr: src_end);
309 u32 *src_end_s = (u32 *)shadow_for(ptr: src_end);
310 dfsan_origin *src_begin_o = origin_for(ptr: (void *)src_a);
311 dfsan_origin *dst = origin_for(ptr: (void *)(end - kOriginAlign));
312 dfsan_origin last_src_o = 0;
313 dfsan_origin last_dst_o = 0;
314 for (; src_end_o >= src_begin_o; --src_end_o, --src_end_s, --dst) {
315 if (!*src_end_s)
316 continue;
317 if (*src_end_o != last_src_o) {
318 last_src_o = *src_end_o;
319 last_dst_o = ChainOrigin(id: last_src_o, stack);
320 }
321 *dst = last_dst_o;
322 }
323 }
324
325 // Copy left unaligned origin if that memory is tainted.
326 if (beg < d)
327 ChainAndWriteOriginIfTainted(src: (uptr)src, size: beg + kOriginAlign - d, dst: beg, stack);
328}
329
330// Copy or move the origins of the len bytes from src to dst. The source and
331// target memory ranges may or may not be overlapped. This is used by memory
332// transfer operations. stack records the stack trace of the memory transfer
333// operation.
334static void MoveOrigin(const void *dst, const void *src, uptr size,
335 StackTrace *stack) {
336 // Validate address regions.
337 if (!MEM_IS_SHADOW(shadow_for(dst)) ||
338 !MEM_IS_SHADOW(shadow_for((void *)((uptr)dst + size))) ||
339 !MEM_IS_SHADOW(shadow_for(src)) ||
340 !MEM_IS_SHADOW(shadow_for((void *)((uptr)src + size)))) {
341 CHECK(false);
342 return;
343 }
344 // If destination origin range overlaps with source origin range, move
345 // origins by copying origins in a reverse order; otherwise, copy origins in
346 // a normal order. The orders of origin transfer are consistent with the
347 // orders of how memcpy and memmove transfer user data.
348 uptr src_aligned_beg = OriginAlignDown(u: (uptr)src);
349 uptr src_aligned_end = OriginAlignDown(u: (uptr)src + size);
350 uptr dst_aligned_beg = OriginAlignDown(u: (uptr)dst);
351 if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
352 return ReverseCopyOrigin(dst, src, size, stack);
353 return CopyOrigin(dst, src, size, stack);
354}
355
356// Set the size bytes from the addres dst to be the origin value.
357static void SetOrigin(const void *dst, uptr size, u32 origin) {
358 if (size == 0)
359 return;
360
361 // Origin mapping is 4 bytes per 4 bytes of application memory.
362 // Here we extend the range such that its left and right bounds are both
363 // 4 byte aligned.
364 uptr x = unaligned_origin_for(ptr: (uptr)dst);
365 uptr beg = OriginAlignDown(u: x);
366 uptr end = OriginAlignUp(u: x + size); // align up.
367 u64 origin64 = ((u64)origin << 32) | origin;
368 // This is like memset, but the value is 32-bit. We unroll by 2 to write
369 // 64 bits at once. May want to unroll further to get 128-bit stores.
370 if (beg & 7ULL) {
371 if (*(u32 *)beg != origin)
372 *(u32 *)beg = origin;
373 beg += 4;
374 }
375 for (uptr addr = beg; addr < (end & ~7UL); addr += 8) {
376 if (*(u64 *)addr == origin64)
377 continue;
378 *(u64 *)addr = origin64;
379 }
380 if (end & 7ULL)
381 if (*(u32 *)(end - kOriginAlign) != origin)
382 *(u32 *)(end - kOriginAlign) = origin;
383}
384
385#define RET_CHAIN_ORIGIN(id) \
386 GET_CALLER_PC_BP; \
387 GET_STORE_STACK_TRACE_PC_BP(pc, bp); \
388 return ChainOrigin(id, &stack);
389
390// Return a new origin chain with the previous ID id and the current stack
391// trace.
392extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
393__dfsan_chain_origin(dfsan_origin id) {
394 RET_CHAIN_ORIGIN(id)
395}
396
397// Return a new origin chain with the previous ID id and the current stack
398// trace if the label is tainted.
399extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
400__dfsan_chain_origin_if_tainted(dfsan_label label, dfsan_origin id) {
401 if (!label)
402 return id;
403 RET_CHAIN_ORIGIN(id)
404}
405
406// Copy or move the origins of the len bytes from src to dst.
407extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_mem_origin_transfer(
408 const void *dst, const void *src, uptr len) {
409 if (src == dst)
410 return;
411 GET_CALLER_PC_BP;
412 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
413 MoveOrigin(dst, src, size: len, stack: &stack);
414}
415
416extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(
417 const void *dst, const void *src, uptr len) {
418 __dfsan_mem_origin_transfer(dst, src, len);
419}
420
421static void CopyShadow(void *dst, const void *src, uptr len) {
422 internal_memcpy(dest: (void *)__dfsan::shadow_for(ptr: dst),
423 src: (const void *)__dfsan::shadow_for(ptr: src),
424 n: len * sizeof(dfsan_label));
425}
426
427extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_shadow_transfer(
428 void *dst, const void *src, uptr len) {
429 CopyShadow(dst, src, len);
430}
431
432// Copy shadow and origins of the len bytes from src to dst.
433extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
434__dfsan_mem_shadow_origin_transfer(void *dst, const void *src, uptr size) {
435 if (src == dst)
436 return;
437 CopyShadow(dst, src, len: size);
438 if (dfsan_get_track_origins()) {
439 // Duplicating code instead of calling __dfsan_mem_origin_transfer
440 // so that the getting the caller stack frame works correctly.
441 GET_CALLER_PC_BP;
442 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
443 MoveOrigin(dst, src, size, stack: &stack);
444 }
445}
446
447// Copy shadow and origins as per __atomic_compare_exchange.
448extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
449__dfsan_mem_shadow_origin_conditional_exchange(u8 condition, void *target,
450 void *expected,
451 const void *desired, uptr size) {
452 void *dst;
453 const void *src;
454 // condition is result of native call to __atomic_compare_exchange
455 if (condition) {
456 // Copy desired into target
457 dst = target;
458 src = desired;
459 } else {
460 // Copy target into expected
461 dst = expected;
462 src = target;
463 }
464 if (src == dst)
465 return;
466 CopyShadow(dst, src, len: size);
467 if (dfsan_get_track_origins()) {
468 // Duplicating code instead of calling __dfsan_mem_origin_transfer
469 // so that the getting the caller stack frame works correctly.
470 GET_CALLER_PC_BP;
471 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
472 MoveOrigin(dst, src, size, stack: &stack);
473 }
474}
475
476bool __dfsan::dfsan_inited;
477bool __dfsan::dfsan_init_is_running;
478
479void __dfsan::dfsan_copy_memory(void *dst, const void *src, uptr size) {
480 internal_memcpy(dest: dst, src, n: size);
481 dfsan_mem_shadow_transfer(dst, src, len: size);
482 if (dfsan_get_track_origins())
483 dfsan_mem_origin_transfer(dst, src, len: size);
484}
485
486// Releases the pages within the origin address range.
487static void ReleaseOrigins(void *addr, uptr size) {
488 const uptr beg_origin_addr = (uptr)__dfsan::origin_for(ptr: addr);
489 const void *end_addr = (void *)((uptr)addr + size);
490 const uptr end_origin_addr = (uptr)__dfsan::origin_for(ptr: end_addr);
491
492 if (end_origin_addr - beg_origin_addr <
493 common_flags()->clear_shadow_mmap_threshold)
494 return;
495
496 const uptr page_size = GetPageSizeCached();
497 const uptr beg_aligned = RoundUpTo(size: beg_origin_addr, boundary: page_size);
498 const uptr end_aligned = RoundDownTo(x: end_origin_addr, boundary: page_size);
499
500 if (!MmapFixedSuperNoReserve(fixed_addr: beg_aligned, size: end_aligned - beg_aligned))
501 Die();
502}
503
504static void WriteZeroShadowInRange(uptr beg, uptr end) {
505 // Don't write the label if it is already the value we need it to be.
506 // In a program where most addresses are not labeled, it is common that
507 // a page of shadow memory is entirely zeroed. The Linux copy-on-write
508 // implementation will share all of the zeroed pages, making a copy of a
509 // page when any value is written. The un-sharing will happen even if
510 // the value written does not change the value in memory. Avoiding the
511 // write when both |label| and |*labelp| are zero dramatically reduces
512 // the amount of real memory used by large programs.
513 if (!mem_is_zero(mem: (const char *)beg, size: end - beg))
514 internal_memset(s: (void *)beg, c: 0, n: end - beg);
515}
516
517// Releases the pages within the shadow address range, and sets
518// the shadow addresses not on the pages to be 0.
519static void ReleaseOrClearShadows(void *addr, uptr size) {
520 const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(ptr: addr);
521 const void *end_addr = (void *)((uptr)addr + size);
522 const uptr end_shadow_addr = (uptr)__dfsan::shadow_for(ptr: end_addr);
523
524 if (end_shadow_addr - beg_shadow_addr <
525 common_flags()->clear_shadow_mmap_threshold) {
526 WriteZeroShadowInRange(beg: beg_shadow_addr, end: end_shadow_addr);
527 return;
528 }
529
530 const uptr page_size = GetPageSizeCached();
531 const uptr beg_aligned = RoundUpTo(size: beg_shadow_addr, boundary: page_size);
532 const uptr end_aligned = RoundDownTo(x: end_shadow_addr, boundary: page_size);
533
534 if (beg_aligned >= end_aligned) {
535 WriteZeroShadowInRange(beg: beg_shadow_addr, end: end_shadow_addr);
536 } else {
537 if (beg_aligned != beg_shadow_addr)
538 WriteZeroShadowInRange(beg: beg_shadow_addr, end: beg_aligned);
539 if (end_aligned != end_shadow_addr)
540 WriteZeroShadowInRange(beg: end_aligned, end: end_shadow_addr);
541 if (!MmapFixedSuperNoReserve(fixed_addr: beg_aligned, size: end_aligned - beg_aligned))
542 Die();
543 }
544}
545
546static void SetShadow(dfsan_label label, void *addr, uptr size,
547 dfsan_origin origin) {
548 if (0 != label) {
549 const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(ptr: addr);
550 internal_memset(s: (void *)beg_shadow_addr, c: label, n: size);
551 if (dfsan_get_track_origins())
552 SetOrigin(dst: addr, size, origin);
553 return;
554 }
555
556 if (dfsan_get_track_origins())
557 ReleaseOrigins(addr, size);
558
559 ReleaseOrClearShadows(addr, size);
560}
561
562// If the label s is tainted, set the size bytes from the address p to be a new
563// origin chain with the previous ID o and the current stack trace. This is
564// used by instrumentation to reduce code size when too much code is inserted.
565extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
566 dfsan_label s, void *p, uptr size, dfsan_origin o) {
567 if (UNLIKELY(s)) {
568 GET_CALLER_PC_BP;
569 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
570 SetOrigin(dst: p, size, origin: ChainOrigin(id: o, stack: &stack));
571 }
572}
573
574extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_set_label(
575 dfsan_label label, dfsan_origin origin, void *addr, uptr size) {
576 SetShadow(label, addr, size, origin);
577}
578
579SANITIZER_INTERFACE_ATTRIBUTE
580void dfsan_set_label(dfsan_label label, void *addr, uptr size) {
581 dfsan_origin init_origin = 0;
582 if (label && dfsan_get_track_origins()) {
583 GET_CALLER_PC_BP;
584 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
585 init_origin = ChainOrigin(id: 0, stack: &stack, from_init: true);
586 }
587 SetShadow(label, addr, size, origin: init_origin);
588}
589
590SANITIZER_INTERFACE_ATTRIBUTE
591void dfsan_add_label(dfsan_label label, void *addr, uptr size) {
592 if (0 == label)
593 return;
594
595 if (dfsan_get_track_origins()) {
596 GET_CALLER_PC_BP;
597 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
598 dfsan_origin init_origin = ChainOrigin(id: 0, stack: &stack, from_init: true);
599 SetOrigin(dst: addr, size, origin: init_origin);
600 }
601
602 for (dfsan_label *labelp = shadow_for(ptr: addr); size != 0; --size, ++labelp)
603 *labelp |= label;
604}
605
606// Unlike the other dfsan interface functions the behavior of this function
607// depends on the label of one of its arguments. Hence it is implemented as a
608// custom function.
609extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
610__dfsw_dfsan_get_label(long data, dfsan_label data_label,
611 dfsan_label *ret_label) {
612 *ret_label = 0;
613 return data_label;
614}
615
616extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfso_dfsan_get_label(
617 long data, dfsan_label data_label, dfsan_label *ret_label,
618 dfsan_origin data_origin, dfsan_origin *ret_origin) {
619 *ret_label = 0;
620 *ret_origin = 0;
621 return data_label;
622}
623
624// This function is used if dfsan_get_origin is called when origin tracking is
625// off.
626extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfsw_dfsan_get_origin(
627 long data, dfsan_label data_label, dfsan_label *ret_label) {
628 *ret_label = 0;
629 return 0;
630}
631
632extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfso_dfsan_get_origin(
633 long data, dfsan_label data_label, dfsan_label *ret_label,
634 dfsan_origin data_origin, dfsan_origin *ret_origin) {
635 *ret_label = 0;
636 *ret_origin = 0;
637 return data_origin;
638}
639
640SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
641dfsan_read_label(const void *addr, uptr size) {
642 if (size == 0)
643 return 0;
644 return __dfsan_union_load(ls: shadow_for(ptr: addr), n: size);
645}
646
647SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
648dfsan_read_origin_of_first_taint(const void *addr, uptr size) {
649 return GetOriginIfTainted(addr: (uptr)addr, size);
650}
651
652SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_label_origin(dfsan_label label,
653 dfsan_origin origin,
654 void *addr,
655 uptr size) {
656 __dfsan_set_label(label, origin, addr, size);
657}
658
659extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
660dfsan_has_label(dfsan_label label, dfsan_label elem) {
661 return (label & elem) == elem;
662}
663
664namespace __dfsan {
665typedef void (*dfsan_conditional_callback_t)(dfsan_label label,
666 dfsan_origin origin);
667
668} // namespace __dfsan
669static dfsan_conditional_callback_t conditional_callback = nullptr;
670static dfsan_label labels_in_signal_conditional = 0;
671
672static void ConditionalCallback(dfsan_label label, dfsan_origin origin) {
673 // Programs have many branches. For efficiency the conditional sink callback
674 // handler needs to ignore as many as possible as early as possible.
675 if (label == 0) {
676 return;
677 }
678 if (conditional_callback == nullptr) {
679 return;
680 }
681
682 // This initial ConditionalCallback handler needs to be in here in dfsan
683 // runtime (rather than being an entirely user implemented hook) so that it
684 // has access to dfsan thread information.
685 DFsanThread *t = GetCurrentThread();
686 // A callback operation which does useful work (like record the flow) will
687 // likely be too long executed in a signal handler.
688 if (t && t->InSignalHandler()) {
689 // Record set of labels used in signal handler for completeness.
690 labels_in_signal_conditional |= label;
691 return;
692 }
693
694 conditional_callback(label, origin);
695}
696
697extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
698__dfsan_conditional_callback_origin(dfsan_label label, dfsan_origin origin) {
699 ConditionalCallback(label, origin);
700}
701
702extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_conditional_callback(
703 dfsan_label label) {
704 ConditionalCallback(label, origin: 0);
705}
706
707extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_conditional_callback(
708 __dfsan::dfsan_conditional_callback_t callback) {
709 conditional_callback = callback;
710}
711
712extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
713dfsan_get_labels_in_signal_conditional() {
714 return labels_in_signal_conditional;
715}
716
717namespace __dfsan {
718typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label,
719 dfsan_origin origin,
720 const char *file,
721 unsigned int line,
722 const char *function);
723
724} // namespace __dfsan
725static dfsan_reaches_function_callback_t reaches_function_callback = nullptr;
726static dfsan_label labels_in_signal_reaches_function = 0;
727
728static void ReachesFunctionCallback(dfsan_label label, dfsan_origin origin,
729 const char *file, unsigned int line,
730 const char *function) {
731 if (label == 0) {
732 return;
733 }
734 if (reaches_function_callback == nullptr) {
735 return;
736 }
737
738 // This initial ReachesFunctionCallback handler needs to be in here in dfsan
739 // runtime (rather than being an entirely user implemented hook) so that it
740 // has access to dfsan thread information.
741 DFsanThread *t = GetCurrentThread();
742 // A callback operation which does useful work (like record the flow) will
743 // likely be too long executed in a signal handler.
744 if (t && t->InSignalHandler()) {
745 // Record set of labels used in signal handler for completeness.
746 labels_in_signal_reaches_function |= label;
747 return;
748 }
749
750 reaches_function_callback(label, origin, file, line, function);
751}
752
753extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
754__dfsan_reaches_function_callback_origin(dfsan_label label, dfsan_origin origin,
755 const char *file, unsigned int line,
756 const char *function) {
757 ReachesFunctionCallback(label, origin, file, line, function);
758}
759
760extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
761__dfsan_reaches_function_callback(dfsan_label label, const char *file,
762 unsigned int line, const char *function) {
763 ReachesFunctionCallback(label, origin: 0, file, line, function);
764}
765
766extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
767dfsan_set_reaches_function_callback(
768 __dfsan::dfsan_reaches_function_callback_t callback) {
769 reaches_function_callback = callback;
770}
771
772extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
773dfsan_get_labels_in_signal_reaches_function() {
774 return labels_in_signal_reaches_function;
775}
776
777namespace {
778class Decorator : public __sanitizer::SanitizerCommonDecorator {
779 public:
780 Decorator() : SanitizerCommonDecorator() {}
781 const char *Origin() const { return Magenta(); }
782};
783} // namespace
784
785static void PrintNoOriginTrackingWarning() {
786 Decorator d;
787 Printf(
788 format: " %sDFSan: origin tracking is not enabled. Did you specify the "
789 "-dfsan-track-origins=1 option?%s\n",
790 d.Warning(), d.Default());
791}
792
793static void PrintNoTaintWarning(const void *address) {
794 Decorator d;
795 Printf(format: " %sDFSan: no tainted value at %x%s\n", d.Warning(), address,
796 d.Default());
797}
798
799static void PrintInvalidOriginWarning(dfsan_label label, const void *address) {
800 Decorator d;
801 Printf(
802 format: " %sTaint value 0x%x (at %p) has invalid origin tracking. This can "
803 "be a DFSan bug.%s\n",
804 d.Warning(), label, address, d.Default());
805}
806
807static void PrintInvalidOriginIdWarning(dfsan_origin origin) {
808 Decorator d;
809 Printf(
810 format: " %sOrigin Id %d has invalid origin tracking. This can "
811 "be a DFSan bug.%s\n",
812 d.Warning(), origin, d.Default());
813}
814
815static bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) {
816 Decorator d;
817 bool found = false;
818
819 while (o.isChainedOrigin()) {
820 StackTrace stack;
821 dfsan_origin origin_id = o.raw_id();
822 o = o.getNextChainedOrigin(stack: &stack);
823 if (o.isChainedOrigin())
824 out->AppendF(
825 format: " %sOrigin value: 0x%x, Taint value was stored to memory at%s\n",
826 d.Origin(), origin_id, d.Default());
827 else
828 out->AppendF(format: " %sOrigin value: 0x%x, Taint value was created at%s\n",
829 d.Origin(), origin_id, d.Default());
830
831 // Includes a trailing newline, so no need to add it again.
832 stack.PrintTo(output: out);
833 found = true;
834 }
835
836 return found;
837}
838
839static bool PrintOriginTraceToStr(const void *addr, const char *description,
840 InternalScopedString *out) {
841 CHECK(out);
842 CHECK(dfsan_get_track_origins());
843 Decorator d;
844
845 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
846 CHECK(label);
847
848 const dfsan_origin origin = *__dfsan::origin_for(ptr: addr);
849
850 out->AppendF(format: " %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
851 d.Origin(), label, addr, description ? description : "",
852 d.Default());
853
854 Origin o = Origin::FromRawId(id: origin);
855 return PrintOriginTraceFramesToStr(o, out);
856}
857
858extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
859 const void *addr, const char *description) {
860 if (!dfsan_get_track_origins()) {
861 PrintNoOriginTrackingWarning();
862 return;
863 }
864
865 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
866 if (!label) {
867 PrintNoTaintWarning(address: addr);
868 return;
869 }
870
871 InternalScopedString trace;
872 bool success = PrintOriginTraceToStr(addr, description, out: &trace);
873
874 if (trace.length())
875 Printf(format: "%s", trace.data());
876
877 if (!success)
878 PrintInvalidOriginWarning(label, address: addr);
879}
880
881extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
882dfsan_sprint_origin_trace(const void *addr, const char *description,
883 char *out_buf, uptr out_buf_size) {
884 CHECK(out_buf);
885
886 if (!dfsan_get_track_origins()) {
887 PrintNoOriginTrackingWarning();
888 return 0;
889 }
890
891 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
892 if (!label) {
893 PrintNoTaintWarning(address: addr);
894 return 0;
895 }
896
897 InternalScopedString trace;
898 bool success = PrintOriginTraceToStr(addr, description, out: &trace);
899
900 if (!success) {
901 PrintInvalidOriginWarning(label, address: addr);
902 return 0;
903 }
904
905 if (out_buf_size) {
906 internal_strncpy(dst: out_buf, src: trace.data(), n: out_buf_size - 1);
907 out_buf[out_buf_size - 1] = '\0';
908 }
909
910 return trace.length();
911}
912
913extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace(
914 dfsan_origin origin) {
915 if (!dfsan_get_track_origins()) {
916 PrintNoOriginTrackingWarning();
917 return;
918 }
919 Origin o = Origin::FromRawId(id: origin);
920
921 InternalScopedString trace;
922 bool success = PrintOriginTraceFramesToStr(o, out: &trace);
923
924 if (trace.length())
925 Printf(format: "%s", trace.data());
926
927 if (!success)
928 PrintInvalidOriginIdWarning(origin);
929}
930
931extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace(
932 dfsan_origin origin, char *out_buf, uptr out_buf_size) {
933 CHECK(out_buf);
934
935 if (!dfsan_get_track_origins()) {
936 PrintNoOriginTrackingWarning();
937 return 0;
938 }
939 Origin o = Origin::FromRawId(id: origin);
940
941 InternalScopedString trace;
942 bool success = PrintOriginTraceFramesToStr(o, out: &trace);
943
944 if (!success) {
945 PrintInvalidOriginIdWarning(origin);
946 return 0;
947 }
948
949 if (out_buf_size) {
950 internal_strncpy(dst: out_buf, src: trace.data(), n: out_buf_size - 1);
951 out_buf[out_buf_size - 1] = '\0';
952 }
953
954 return trace.length();
955}
956
957extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
958dfsan_get_init_origin(const void *addr) {
959 if (!dfsan_get_track_origins())
960 return 0;
961
962 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
963 if (!label)
964 return 0;
965
966 const dfsan_origin origin = *__dfsan::origin_for(ptr: addr);
967
968 Origin o = Origin::FromRawId(id: origin);
969 dfsan_origin origin_id = o.raw_id();
970 while (o.isChainedOrigin()) {
971 StackTrace stack;
972 origin_id = o.raw_id();
973 o = o.getNextChainedOrigin(stack: &stack);
974 }
975 return origin_id;
976}
977
978void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
979 void *context,
980 bool request_fast,
981 u32 max_depth) {
982 using namespace __dfsan;
983 DFsanThread *t = GetCurrentThread();
984 if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast)) {
985 return Unwind(max_depth, pc, bp, context, stack_top: 0, stack_bottom: 0, request_fast_unwind: false);
986 }
987 Unwind(max_depth, pc, bp, context: nullptr, stack_top: t->stack_top(), stack_bottom: t->stack_bottom(), request_fast_unwind: true);
988}
989
990extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_stack_trace() {
991 GET_CALLER_PC_BP;
992 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
993 stack.Print();
994}
995
996extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
997dfsan_sprint_stack_trace(char *out_buf, uptr out_buf_size) {
998 CHECK(out_buf);
999 GET_CALLER_PC_BP;
1000 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
1001 return stack.PrintTo(out_buf, out_buf_size);
1002}
1003
1004void Flags::SetDefaults() {
1005#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
1006#include "dfsan_flags.inc"
1007#undef DFSAN_FLAG
1008}
1009
1010static void RegisterDfsanFlags(FlagParser *parser, Flags *f) {
1011#define DFSAN_FLAG(Type, Name, DefaultValue, Description) \
1012 RegisterFlag(parser, #Name, Description, &f->Name);
1013#include "dfsan_flags.inc"
1014#undef DFSAN_FLAG
1015}
1016
1017static void InitializeFlags() {
1018 SetCommonFlagsDefaults();
1019 {
1020 CommonFlags cf;
1021 cf.CopyFrom(other: *common_flags());
1022 cf.intercept_tls_get_addr = true;
1023 OverrideCommonFlags(cf);
1024 }
1025 flags().SetDefaults();
1026
1027 FlagParser parser;
1028 RegisterCommonFlags(parser: &parser);
1029 RegisterDfsanFlags(parser: &parser, f: &flags());
1030 parser.ParseStringFromEnv(env_name: "DFSAN_OPTIONS");
1031 InitializeCommonFlags();
1032 if (Verbosity()) ReportUnrecognizedFlags();
1033 if (common_flags()->help) parser.PrintFlagDescriptions();
1034}
1035
1036SANITIZER_INTERFACE_ATTRIBUTE
1037void dfsan_clear_arg_tls(uptr offset, uptr size) {
1038 internal_memset(s: (void *)((uptr)__dfsan_arg_tls + offset), c: 0, n: size);
1039}
1040
1041SANITIZER_INTERFACE_ATTRIBUTE
1042void dfsan_clear_thread_local_state() {
1043 internal_memset(s: __dfsan_arg_tls, c: 0, n: sizeof(__dfsan_arg_tls));
1044 internal_memset(s: __dfsan_retval_tls, c: 0, n: sizeof(__dfsan_retval_tls));
1045
1046 if (dfsan_get_track_origins()) {
1047 internal_memset(s: __dfsan_arg_origin_tls, c: 0, n: sizeof(__dfsan_arg_origin_tls));
1048 internal_memset(s: &__dfsan_retval_origin_tls, c: 0,
1049 n: sizeof(__dfsan_retval_origin_tls));
1050 }
1051}
1052
1053SANITIZER_INTERFACE_ATTRIBUTE
1054void dfsan_set_arg_tls(uptr offset, dfsan_label label) {
1055 // 2x to match ShadowTLSAlignment.
1056 // ShadowTLSAlignment should probably be changed.
1057 // TODO: Consider reducing ShadowTLSAlignment to 1.
1058 // Aligning to 2 bytes is probably a remnant of fast16 mode.
1059 ((dfsan_label *)__dfsan_arg_tls)[offset * 2] = label;
1060}
1061
1062SANITIZER_INTERFACE_ATTRIBUTE
1063void dfsan_set_arg_origin_tls(uptr offset, dfsan_origin o) {
1064 __dfsan_arg_origin_tls[offset] = o;
1065}
1066
1067extern "C" void dfsan_flush() {
1068 const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1069 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1070 uptr start = kMemoryLayout[i].start;
1071 uptr end = kMemoryLayout[i].end;
1072 uptr size = end - start;
1073 MappingDesc::Type type = kMemoryLayout[i].type;
1074
1075 if (type != MappingDesc::SHADOW && type != MappingDesc::ORIGIN)
1076 continue;
1077
1078 // Check if the segment should be mapped based on platform constraints.
1079 if (start >= maxVirtualAddress)
1080 continue;
1081
1082 if (!MmapFixedSuperNoReserve(fixed_addr: start, size, name: kMemoryLayout[i].name)) {
1083 Printf(format: "FATAL: DataFlowSanitizer: failed to clear memory region\n");
1084 Die();
1085 }
1086 }
1087 labels_in_signal_conditional = 0;
1088 labels_in_signal_reaches_function = 0;
1089}
1090
1091// TODO: CheckMemoryLayoutSanity is based on msan.
1092// Consider refactoring these into a shared implementation.
1093static void CheckMemoryLayoutSanity() {
1094 uptr prev_end = 0;
1095 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1096 uptr start = kMemoryLayout[i].start;
1097 uptr end = kMemoryLayout[i].end;
1098 MappingDesc::Type type = kMemoryLayout[i].type;
1099 CHECK_LT(start, end);
1100 CHECK_EQ(prev_end, start);
1101 CHECK(addr_is_type(start, type));
1102 CHECK(addr_is_type((start + end) / 2, type));
1103 CHECK(addr_is_type(end - 1, type));
1104 if (type == MappingDesc::APP) {
1105 uptr addr = start;
1106 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1107 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1108 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1109
1110 addr = (start + end) / 2;
1111 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1112 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1113 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1114
1115 addr = end - 1;
1116 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1117 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1118 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1119 }
1120 prev_end = end;
1121 }
1122}
1123
1124// TODO: CheckMemoryRangeAvailability is based on msan.
1125// Consider refactoring these into a shared implementation.
1126static bool CheckMemoryRangeAvailability(uptr beg, uptr size, bool verbose) {
1127 if (size > 0) {
1128 uptr end = beg + size - 1;
1129 if (!MemoryRangeIsAvailable(range_start: beg, range_end: end)) {
1130 if (verbose)
1131 Printf(format: "FATAL: Memory range %p - %p is not available.\n", beg, end);
1132 return false;
1133 }
1134 }
1135 return true;
1136}
1137
1138// TODO: ProtectMemoryRange is based on msan.
1139// Consider refactoring these into a shared implementation.
1140static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) {
1141 if (size > 0) {
1142 void *addr = MmapFixedNoAccess(fixed_addr: beg, size, name);
1143 if (beg == 0 && addr) {
1144 // Depending on the kernel configuration, we may not be able to protect
1145 // the page at address zero.
1146 uptr gap = 16 * GetPageSizeCached();
1147 beg += gap;
1148 size -= gap;
1149 addr = MmapFixedNoAccess(fixed_addr: beg, size, name);
1150 }
1151 if ((uptr)addr != beg) {
1152 uptr end = beg + size - 1;
1153 Printf(format: "FATAL: Cannot protect memory range %p - %p (%s).\n", beg, end,
1154 name);
1155 return false;
1156 }
1157 }
1158 return true;
1159}
1160
1161// TODO: InitShadow is based on msan.
1162// Consider refactoring these into a shared implementation.
1163static bool InitShadow(bool init_origins, bool dry_run) {
1164 // Let user know mapping parameters first.
1165 VPrintf(1, "dfsan_init %p\n", (void *)&__dfsan::dfsan_init);
1166 for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
1167 VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
1168 kMemoryLayout[i].end - 1);
1169
1170 CheckMemoryLayoutSanity();
1171
1172 if (!MEM_IS_APP(&__dfsan::dfsan_init)) {
1173 if (!dry_run)
1174 Printf(format: "FATAL: Code %p is out of application range. Non-PIE build?\n",
1175 (uptr)&__dfsan::dfsan_init);
1176 return false;
1177 }
1178
1179 const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1180
1181 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1182 uptr start = kMemoryLayout[i].start;
1183 uptr end = kMemoryLayout[i].end;
1184 uptr size = end - start;
1185 MappingDesc::Type type = kMemoryLayout[i].type;
1186
1187 // Check if the segment should be mapped based on platform constraints.
1188 if (start >= maxVirtualAddress)
1189 continue;
1190
1191 bool map = type == MappingDesc::SHADOW ||
1192 (init_origins && type == MappingDesc::ORIGIN);
1193 bool protect = type == MappingDesc::INVALID ||
1194 (!init_origins && type == MappingDesc::ORIGIN);
1195 CHECK(!(map && protect));
1196 if (!map && !protect) {
1197 CHECK(type == MappingDesc::APP || type == MappingDesc::ALLOCATOR);
1198
1199 if (dry_run && type == MappingDesc::ALLOCATOR &&
1200 !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1201 return false;
1202 }
1203 if (map) {
1204 if (dry_run && !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1205 return false;
1206 if (!dry_run &&
1207 !MmapFixedSuperNoReserve(fixed_addr: start, size, name: kMemoryLayout[i].name))
1208 return false;
1209 if (!dry_run && common_flags()->use_madv_dontdump)
1210 DontDumpShadowMemory(addr: start, length: size);
1211 }
1212 if (protect) {
1213 if (dry_run && !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1214 return false;
1215 if (!dry_run && !ProtectMemoryRange(beg: start, size, name: kMemoryLayout[i].name))
1216 return false;
1217 }
1218 }
1219
1220 return true;
1221}
1222
1223static bool InitShadowWithReExec(bool init_origins) {
1224 // Start with dry run: check layout is ok, but don't print warnings because
1225 // warning messages will cause tests to fail (even if we successfully re-exec
1226 // after the warning).
1227 bool success = InitShadow(init_origins, dry_run: true);
1228 if (!success) {
1229#if SANITIZER_LINUX
1230 // Perhaps ASLR entropy is too high. If ASLR is enabled, re-exec without it.
1231 int old_personality = personality(persona: 0xffffffff);
1232 bool aslr_on =
1233 (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
1234
1235 if (aslr_on) {
1236 VReport(1,
1237 "WARNING: DataflowSanitizer: memory layout is incompatible, "
1238 "possibly due to high-entropy ASLR.\n"
1239 "Re-execing with fixed virtual address space.\n"
1240 "N.B. reducing ASLR entropy is preferable.\n");
1241 CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
1242 ReExec();
1243 }
1244#endif
1245 }
1246
1247 // The earlier dry run didn't actually map or protect anything. Run again in
1248 // non-dry run mode.
1249 return success && InitShadow(init_origins, dry_run: false);
1250}
1251
1252static void DFsanInit(int argc, char **argv, char **envp) {
1253 CHECK(!dfsan_init_is_running);
1254 if (dfsan_inited)
1255 return;
1256 dfsan_init_is_running = true;
1257 SanitizerToolName = "DataflowSanitizer";
1258
1259 AvoidCVE_2016_2143();
1260
1261 InitializeFlags();
1262
1263 CheckASLR();
1264
1265 InitializePlatformEarly();
1266
1267 if (!InitShadowWithReExec(init_origins: dfsan_get_track_origins())) {
1268 Printf(format: "FATAL: DataflowSanitizer can not mmap the shadow memory.\n");
1269 DumpProcessMap();
1270 Die();
1271 }
1272
1273 initialize_interceptors();
1274
1275 // Set up threads
1276 DFsanTSDInit(destructor: DFsanTSDDtor);
1277
1278 dfsan_allocator_init();
1279
1280 DFsanThread *main_thread = DFsanThread::Create(start_routine: nullptr, arg: nullptr);
1281 SetCurrentThread(main_thread);
1282 main_thread->Init();
1283
1284 dfsan_init_is_running = false;
1285 dfsan_inited = true;
1286}
1287
1288void __dfsan::dfsan_init() { DFsanInit(argc: 0, argv: nullptr, envp: nullptr); }
1289
1290#if SANITIZER_CAN_USE_PREINIT_ARRAY
1291__attribute__((section(".preinit_array"),
1292 used)) static void (*dfsan_init_ptr)(int, char **,
1293 char **) = DFsanInit;
1294#endif
1295