1//===-- nsan.cc -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// NumericalStabilitySanitizer runtime.
10//
11// This implements:
12// - The public nsan interface (include/sanitizer/nsan_interface.h).
13// - The private nsan interface (./nsan.h).
14// - The internal instrumentation interface. These are function emitted by the
15// instrumentation pass:
16// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load
17// These return the shadow memory pointer for loading the shadow value,
18// after checking that the types are consistent. If the types are not
19// consistent, returns nullptr.
20// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store
21// Sets the shadow types appropriately and returns the shadow memory
22// pointer for storing the shadow value.
23// * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the
24// accuracy of a value against its shadow and emits a warning depending
25// on the runtime configuration. The middle part indicates the type of
26// the application value, the suffix (f,d,l) indicates the type of the
27// shadow, and depends on the instrumentation configuration.
28// * __nsan_fcmp_fail_* emits a warning for a fcmp instruction whose
29// corresponding shadow fcmp result differs.
30//
31//===----------------------------------------------------------------------===//
32
33#include "nsan.h"
34#include "nsan_flags.h"
35#include "nsan_stats.h"
36#include "nsan_suppressions.h"
37#include "nsan_thread.h"
38
39#include <assert.h>
40#include <math.h>
41#include <stdint.h>
42#include <stdio.h>
43#include <stdlib.h>
44
45#include "sanitizer_common/sanitizer_atomic.h"
46#include "sanitizer_common/sanitizer_common.h"
47#include "sanitizer_common/sanitizer_libc.h"
48#include "sanitizer_common/sanitizer_report_decorator.h"
49#include "sanitizer_common/sanitizer_stacktrace.h"
50#include "sanitizer_common/sanitizer_symbolizer.h"
51
52using namespace __sanitizer;
53using namespace __nsan;
54
55constexpr int kMaxVectorWidth = 8;
56
57// When copying application memory, we also copy its shadow and shadow type.
58extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
59__nsan_copy_values(const void *daddr, const void *saddr, uptr size) {
60 internal_memmove(dest: GetShadowTypeAddrFor(ptr: daddr), src: GetShadowTypeAddrFor(ptr: saddr),
61 n: size);
62 internal_memmove(dest: GetShadowAddrFor(ptr: daddr), src: GetShadowAddrFor(ptr: saddr),
63 n: size * kShadowScale);
64}
65
66#define NSAN_COPY_VALUES_N(N) \
67 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_copy_##N( \
68 const u8 *daddr, const u8 *saddr) { \
69 __builtin_memmove(GetShadowTypeAddrFor(daddr), \
70 GetShadowTypeAddrFor(saddr), N); \
71 __builtin_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr), \
72 N *kShadowScale); \
73 }
74
75NSAN_COPY_VALUES_N(4)
76NSAN_COPY_VALUES_N(8)
77NSAN_COPY_VALUES_N(16)
78
79extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
80__nsan_set_value_unknown(const void *addr, uptr size) {
81 internal_memset(s: GetShadowTypeAddrFor(ptr: addr), c: 0, n: size);
82}
83
84#define NSAN_SET_VALUE_UNKNOWN_N(N) \
85 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_set_value_unknown_##N( \
86 const u8 *daddr) { \
87 __builtin_memset(GetShadowTypeAddrFor(daddr), 0, N); \
88 }
89
90NSAN_SET_VALUE_UNKNOWN_N(4)
91NSAN_SET_VALUE_UNKNOWN_N(8)
92NSAN_SET_VALUE_UNKNOWN_N(16)
93
94const char *FTInfo<float>::kCppTypeName = "float";
95const char *FTInfo<double>::kCppTypeName = "double";
96const char *FTInfo<long double>::kCppTypeName = "long double";
97const char *FTInfo<__float128>::kCppTypeName = "__float128";
98
99const char FTInfo<float>::kTypePattern[sizeof(float)];
100const char FTInfo<double>::kTypePattern[sizeof(double)];
101const char FTInfo<long double>::kTypePattern[sizeof(long double)];
102
103// Helper for __nsan_dump_shadow_mem: Reads the value at address `ptr`,
104// identified by its type id.
105template <typename ShadowFT>
106static __float128 ReadShadowInternal(const u8 *ptr) {
107 ShadowFT Shadow;
108 __builtin_memcpy(&Shadow, ptr, sizeof(Shadow));
109 return Shadow;
110}
111
112static __float128 ReadShadow(const u8 *ptr, const char ShadowTypeId) {
113 switch (ShadowTypeId) {
114 case 'd':
115 return ReadShadowInternal<double>(ptr);
116 case 'l':
117 return ReadShadowInternal<long double>(ptr);
118 case 'q':
119 return ReadShadowInternal<__float128>(ptr);
120 default:
121 return 0.0;
122 }
123}
124
125namespace {
126class Decorator : public __sanitizer::SanitizerCommonDecorator {
127public:
128 Decorator() : SanitizerCommonDecorator() {}
129 const char *Warning() { return Red(); }
130 const char *Name() { return Green(); }
131 const char *End() { return Default(); }
132};
133
134// Workaround for the fact that Printf() does not support floats.
135struct PrintBuffer {
136 char Buffer[64];
137};
138template <typename FT> struct FTPrinter {};
139
140template <> struct FTPrinter<double> {
141 static PrintBuffer dec(double value) {
142 PrintBuffer result;
143 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20f", value);
144 return result;
145 }
146 static PrintBuffer hex(double value) {
147 PrintBuffer result;
148 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20a", value);
149 return result;
150 }
151};
152
153template <> struct FTPrinter<float> : FTPrinter<double> {};
154
155template <> struct FTPrinter<long double> {
156 static PrintBuffer dec(long double value) {
157 PrintBuffer result;
158 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20Lf", value);
159 return result;
160 }
161 static PrintBuffer hex(long double value) {
162 PrintBuffer result;
163 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20La", value);
164 return result;
165 }
166};
167
168// FIXME: print with full precision.
169template <> struct FTPrinter<__float128> : FTPrinter<long double> {};
170
171// This is a template so that there are no implicit conversions.
172template <typename FT> inline FT ftAbs(FT v);
173
174template <> inline long double ftAbs(long double v) { return fabsl(x: v); }
175template <> inline double ftAbs(double v) { return fabs(x: v); }
176
177// We don't care about nans.
178// std::abs(__float128) code is suboptimal and generates a function call to
179// __getf2().
180template <typename FT> inline FT ftAbs(FT v) { return v >= FT{0} ? v : -v; }
181
182template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {
183 using type = FT2;
184};
185
186template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {
187 using type = FT1;
188};
189
190template <typename FT1, typename FT2>
191using LargestFT =
192 typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;
193
194template <typename T> T max(T a, T b) { return a < b ? b : a; }
195
196} // end anonymous namespace
197
198void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
199 void *context,
200 bool request_fast,
201 u32 max_depth) {
202 using namespace __nsan;
203 NsanThread *t = GetCurrentThread();
204 if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast))
205 return Unwind(max_depth, pc, bp, context, stack_top: t ? t->stack_top() : 0,
206 stack_bottom: t ? t->stack_bottom() : 0, request_fast_unwind: false);
207 if (StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast))
208 Unwind(max_depth, pc, bp, context: nullptr, stack_top: t->stack_top(), stack_bottom: t->stack_bottom(), request_fast_unwind: true);
209 else
210 Unwind(max_depth, pc, bp: 0, context, stack_top: 0, stack_bottom: 0, request_fast_unwind: false);
211}
212
213extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {
214 if (nsan_stats)
215 nsan_stats->Print();
216}
217
218static void NsanAtexit() {
219 Printf(format: "Numerical Sanitizer exit stats:\n");
220 __nsan_print_accumulated_stats();
221 nsan_stats = nullptr;
222}
223
224// The next three functions return a pointer for storing a shadow value for `n`
225// values, after setting the shadow types. We return the pointer instead of
226// storing ourselves because it avoids having to rely on the calling convention
227// around long double being the same for nsan and the target application.
228// We have to have 3 versions because we need to know which type we are storing
229// since we are setting the type shadow memory.
230template <typename FT> static u8 *getShadowPtrForStore(u8 *store_addr, uptr n) {
231 unsigned char *shadow_type = GetShadowTypeAddrFor(ptr: store_addr);
232 for (uptr i = 0; i < n; ++i) {
233 __builtin_memcpy(shadow_type + i * sizeof(FT), FTInfo<FT>::kTypePattern,
234 sizeof(FTInfo<FT>::kTypePattern));
235 }
236 return GetShadowAddrFor(ptr: store_addr);
237}
238
239extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
240__nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {
241 return getShadowPtrForStore<float>(store_addr, n);
242}
243
244extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
245__nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {
246 return getShadowPtrForStore<double>(store_addr, n);
247}
248
249extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
250__nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {
251 return getShadowPtrForStore<long double>(store_addr, n);
252}
253
254template <typename FT> static bool IsValidShadowType(const u8 *shadow_type) {
255 return __builtin_memcmp(shadow_type, FTInfo<FT>::kTypePattern, sizeof(FT)) ==
256 0;
257}
258
259template <int kSize, typename T> static bool IsZero(const T *ptr) {
260 constexpr const char kZeros[kSize] = {}; // Zero initialized.
261 return __builtin_memcmp(ptr, kZeros, kSize) == 0;
262}
263
264template <typename FT> static bool IsUnknownShadowType(const u8 *shadow_type) {
265 return IsZero<sizeof(FTInfo<FT>::kTypePattern)>(shadow_type);
266}
267
268// The three folowing functions check that the address stores a complete
269// shadow value of the given type and return a pointer for loading.
270// They return nullptr if the type of the value is unknown or incomplete.
271template <typename FT>
272static const u8 *getShadowPtrForLoad(const u8 *load_addr, uptr n) {
273 const u8 *const shadow_type = GetShadowTypeAddrFor(ptr: load_addr);
274 for (uptr i = 0; i < n; ++i) {
275 if (!IsValidShadowType<FT>(shadow_type + i * sizeof(FT))) {
276 // If loadtracking stats are enabled, log loads with invalid types
277 // (tampered with through type punning).
278 if (flags().enable_loadtracking_stats) {
279 if (IsUnknownShadowType<FT>(shadow_type + i * sizeof(FT))) {
280 // Warn only if the value is non-zero. Zero is special because
281 // applications typically initialize large buffers to zero in an
282 // untyped way.
283 if (!IsZero<sizeof(FT)>(load_addr)) {
284 GET_CALLER_PC_BP;
285 nsan_stats->AddUnknownLoadTrackingEvent(pc, bp);
286 }
287 } else {
288 GET_CALLER_PC_BP;
289 nsan_stats->AddInvalidLoadTrackingEvent(pc, bp);
290 }
291 }
292 return nullptr;
293 }
294 }
295 return GetShadowAddrFor(ptr: load_addr);
296}
297
298extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
299__nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {
300 return getShadowPtrForLoad<float>(load_addr, n);
301}
302
303extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
304__nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {
305 return getShadowPtrForLoad<double>(load_addr, n);
306}
307
308extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
309__nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {
310 return getShadowPtrForLoad<long double>(load_addr, n);
311}
312
313// Returns the raw shadow pointer. The returned pointer should be considered
314// opaque.
315extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
316__nsan_internal_get_raw_shadow_ptr(const u8 *addr) {
317 return GetShadowAddrFor(ptr: addr);
318}
319
320// Returns the raw shadow type pointer. The returned pointer should be
321// considered opaque.
322extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
323__nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {
324 return reinterpret_cast<u8 *>(GetShadowTypeAddrFor(ptr: addr));
325}
326
327static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }
328
329static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }
330
331// Checks the consistency of the value types at the given type pointer.
332// If the value is inconsistent, returns ValueType::kUnknown. Else, return the
333// consistent type.
334template <typename FT>
335static bool checkValueConsistency(const u8 *shadow_type) {
336 const int pos = getValuePos(c: *shadow_type);
337 // Check that all bytes from the start of the value are ordered.
338 for (uptr i = 0; i < sizeof(FT); ++i) {
339 const u8 T = *(shadow_type - pos + i);
340 if (!(getValueType(c: T) == FTInfo<FT>::kValueType && getValuePos(c: T) == i))
341 return false;
342 }
343 return true;
344}
345
346// The instrumentation automatically appends `shadow_value_type_ids`, see
347// maybeAddSuffixForNsanInterface.
348extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
349__nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,
350 size_t shadow_value_type_ids) {
351 const u8 *const shadow_type = GetShadowTypeAddrFor(ptr: addr);
352 const u8 *const shadow = GetShadowAddrFor(ptr: addr);
353
354 constexpr int kMaxNumDecodedValues = 16;
355 __float128 decoded_values[kMaxNumDecodedValues];
356 int num_decoded_values = 0;
357 if (bytes_per_line > 4 * kMaxNumDecodedValues)
358 bytes_per_line = 4 * kMaxNumDecodedValues;
359
360 // We keep track of the current type and position as we go.
361 ValueType LastValueTy = kUnknownValueType;
362 int LastPos = -1;
363 size_t Offset = 0;
364 for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;
365 ++R) {
366 printf(format: "%p: ", (void *)(addr + R * bytes_per_line));
367 for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {
368 const ValueType ValueTy = getValueType(c: shadow_type[Offset]);
369 const int pos = getValuePos(c: shadow_type[Offset]);
370 if (ValueTy == LastValueTy && pos == LastPos + 1) {
371 ++LastPos;
372 } else {
373 LastValueTy = ValueTy;
374 LastPos = pos == 0 ? 0 : -1;
375 }
376
377 switch (ValueTy) {
378 case kUnknownValueType:
379 printf(format: "__ ");
380 break;
381 case kFloatValueType:
382 printf(format: "f%x ", pos);
383 if (LastPos == sizeof(float) - 1) {
384 decoded_values[num_decoded_values] =
385 ReadShadow(ptr: shadow + kShadowScale * (Offset + 1 - sizeof(float)),
386 ShadowTypeId: static_cast<char>(shadow_value_type_ids & 0xff));
387 ++num_decoded_values;
388 }
389 break;
390 case kDoubleValueType:
391 printf(format: "d%x ", pos);
392 if (LastPos == sizeof(double) - 1) {
393 decoded_values[num_decoded_values] = ReadShadow(
394 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(double)),
395 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 8) & 0xff));
396 ++num_decoded_values;
397 }
398 break;
399 case kFp80ValueType:
400 printf(format: "l%x ", pos);
401 if (LastPos == sizeof(long double) - 1) {
402 decoded_values[num_decoded_values] = ReadShadow(
403 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(long double)),
404 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 16) & 0xff));
405 ++num_decoded_values;
406 }
407 break;
408 }
409 ++Offset;
410 }
411 for (int i = 0; i < num_decoded_values; ++i) {
412 printf(format: " (%s)", FTPrinter<__float128>::dec(value: decoded_values[i]).Buffer);
413 }
414 num_decoded_values = 0;
415 printf(format: "\n");
416 }
417}
418
419alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
420 thread_local uptr __nsan_shadow_ret_tag = 0;
421
422alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
423 thread_local char __nsan_shadow_ret_ptr[kMaxVectorWidth *
424 sizeof(__float128)];
425
426alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
427 thread_local uptr __nsan_shadow_args_tag = 0;
428
429// Maximum number of args. This should be enough for anyone (tm). An alternate
430// scheme is to have the generated code create an alloca and make
431// __nsan_shadow_args_ptr point ot the alloca.
432constexpr const int kMaxNumArgs = 128;
433alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
434 thread_local char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs *
435 sizeof(__float128)];
436
437enum ContinuationType { // Keep in sync with instrumentation pass.
438 kContinueWithShadow = 0,
439 kResumeFromValue = 1,
440};
441
442// Checks the consistency between application and shadow value. Returns true
443// when the instrumented code should resume computations from the original value
444// rather than the shadow value. This prevents one error to propagate to all
445// subsequent operations. This behaviour is tunable with flags.
446template <typename FT, typename ShadowFT>
447int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,
448 uptr CheckArg) {
449 // We do all comparisons in the InternalFT domain, which is the largest FT
450 // type.
451 using InternalFT = LargestFT<FT, ShadowFT>;
452 const InternalFT check_value = value;
453 const InternalFT check_shadow = Shadow;
454
455 // We only check for NaNs in the value, not the shadow.
456 if (flags().check_nan && isnan(value)) {
457 GET_CALLER_PC_BP;
458 BufferedStackTrace stack;
459 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
460 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Consistency)) {
461 // FIXME: optionally print.
462 return flags().resume_after_suppression ? kResumeFromValue
463 : kContinueWithShadow;
464 }
465 Decorator D;
466 Printf(format: "%s", D.Warning());
467 Printf(format: "WARNING: NumericalStabilitySanitizer: NaN detected\n");
468 Printf(format: "%s", D.Default());
469 stack.Print();
470 if (flags().halt_on_error) {
471 if (common_flags()->abort_on_error)
472 Printf(format: "ABORTING\n");
473 else
474 Printf(format: "Exiting\n");
475 Die();
476 }
477 // Performing other tests for NaN values is meaningless when dealing with numbers.
478 return kResumeFromValue;
479 }
480
481 // See this article for an interesting discussion of how to compare floats:
482 // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
483 static constexpr const FT Eps = FTInfo<FT>::kEpsilon;
484
485 const InternalFT abs_err = ftAbs(check_value - check_shadow);
486
487 if (flags().enable_check_stats) {
488 GET_CALLER_PC_BP;
489 // We are re-computing `largest` here because this is a cold branch, and we
490 // want to avoid having to move the computation of `largest` before the
491 // absolute value check when this branch is not taken.
492 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
493 nsan_stats->AddCheck(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
494 }
495
496 // Note: writing the comparison that way ensures that when `abs_err` is Nan
497 // (value and shadow are inf or -inf), we pass the test.
498 if (!(abs_err >= flags().cached_absolute_error_threshold))
499 return kContinueWithShadow;
500
501 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
502 if (abs_err * (1ull << flags().log2_max_relative_error) <= largest)
503 return kContinueWithShadow; // No problem here.
504
505 if (!flags().disable_warnings) {
506 GET_CALLER_PC_BP;
507 UNINITIALIZED BufferedStackTrace stack;
508 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
509 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Consistency)) {
510 // FIXME: optionally print.
511 return flags().resume_after_suppression ? kResumeFromValue
512 : kContinueWithShadow;
513 }
514
515 Decorator D;
516 Printf(format: "%s", D.Warning());
517 // Printf does not support float formatting.
518 char RelErrBuf[64] = "inf";
519 if (largest > Eps) {
520 snprintf(s: RelErrBuf, maxlen: sizeof(RelErrBuf) - 1, format: "%.20Lf%% (2^%.0Lf epsilons)",
521 static_cast<long double>(100.0 * abs_err / largest),
522 log2l(x: static_cast<long double>(abs_err / largest / Eps)));
523 }
524 char ulp_err_buf[128] = "";
525 const double shadow_ulp_diff = GetULPDiff(check_value, check_shadow);
526 if (shadow_ulp_diff != kMaxULPDiff) {
527 // This is the ULP diff in the internal domain. The user actually cares
528 // about that in the original domain.
529 const double ulp_diff =
530 shadow_ulp_diff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -
531 FTInfo<FT>::kMantissaBits));
532 snprintf(s: ulp_err_buf, maxlen: sizeof(ulp_err_buf) - 1,
533 format: "(%.0f ULPs == %.1f digits == %.1f bits)", ulp_diff,
534 log10(x: ulp_diff), log2(x: ulp_diff));
535 }
536 Printf(format: "WARNING: NumericalStabilitySanitizer: inconsistent shadow results");
537 switch (CheckType) {
538 case CheckTypeT::kUnknown:
539 case CheckTypeT::kFcmp:
540 case CheckTypeT::kMaxCheckType:
541 break;
542 case CheckTypeT::kRet:
543 Printf(format: " while checking return value");
544 break;
545 case CheckTypeT::kArg:
546 Printf(format: " while checking call argument #%d", static_cast<int>(CheckArg));
547 break;
548 case CheckTypeT::kLoad:
549 Printf(
550 format: " while checking load from address 0x%lx. This is due to incorrect "
551 "shadow memory tracking, typically due to uninstrumented code "
552 "writing to memory.",
553 CheckArg);
554 break;
555 case CheckTypeT::kStore:
556 Printf(format: " while checking store to address 0x%lx", CheckArg);
557 break;
558 case CheckTypeT::kInsert:
559 Printf(format: " while checking vector insert");
560 break;
561 case CheckTypeT::kUser:
562 Printf(format: " in user-initiated check");
563 break;
564 }
565 using ValuePrinter = FTPrinter<FT>;
566 using ShadowPrinter = FTPrinter<ShadowFT>;
567 Printf(format: "%s", D.Default());
568
569 Printf("\n"
570 "%-12s precision (native): dec: %s hex: %s\n"
571 "%-12s precision (shadow): dec: %s hex: %s\n"
572 "shadow truncated to %-12s: dec: %s hex: %s\n"
573 "Relative error: %s\n"
574 "Absolute error: %s\n"
575 "%s\n",
576 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(value).Buffer,
577 ValuePrinter::hex(value).Buffer, FTInfo<ShadowFT>::kCppTypeName,
578 ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,
579 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,
580 ValuePrinter::hex(Shadow).Buffer, RelErrBuf,
581 ValuePrinter::hex(abs_err).Buffer, ulp_err_buf);
582 stack.Print();
583 }
584
585 if (flags().enable_warning_stats) {
586 GET_CALLER_PC_BP;
587 nsan_stats->AddWarning(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
588 }
589
590 if (flags().halt_on_error) {
591 if (common_flags()->abort_on_error)
592 Printf(format: "ABORTING\n");
593 else
594 Printf(format: "Exiting\n");
595 Die();
596 }
597 return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
598}
599
600extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(
601 float value, double shadow, int32_t check_type, uptr check_arg) {
602 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
603}
604
605extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(
606 double value, long double shadow, int32_t check_type, uptr check_arg) {
607 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
608}
609
610extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(
611 double value, __float128 shadow, int32_t check_type, uptr check_arg) {
612 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
613}
614
615extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t
616__nsan_internal_check_longdouble_q(long double value, __float128 shadow,
617 int32_t check_type, uptr check_arg) {
618 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
619}
620
621static const char *GetTruthValueName(bool v) { return v ? "true" : "false"; }
622
623// This uses the same values as CmpInst::Predicate.
624static const char *GetPredicateName(int v) {
625 switch (v) {
626 case 0:
627 return "(false)";
628 case 1:
629 return "==";
630 case 2:
631 return ">";
632 case 3:
633 return ">=";
634 case 4:
635 return "<";
636 case 5:
637 return "<=";
638 case 6:
639 return "!=";
640 case 7:
641 return "(ordered)";
642 case 8:
643 return "(unordered)";
644 case 9:
645 return "==";
646 case 10:
647 return ">";
648 case 11:
649 return ">=";
650 case 12:
651 return "<";
652 case 13:
653 return "<=";
654 case 14:
655 return "!=";
656 case 15:
657 return "(true)";
658 }
659 return "??";
660}
661
662template <typename FT, typename ShadowFT>
663void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,
664 ShadowFT RhsShadow, int Predicate, bool result,
665 bool ShadowResult) {
666 if (result == ShadowResult) {
667 // When a vector comparison fails, we fail each element of the comparison
668 // to simplify instrumented code. Skip elements where the shadow comparison
669 // gave the same result as the original one.
670 return;
671 }
672
673 GET_CALLER_PC_BP;
674 UNINITIALIZED BufferedStackTrace stack;
675 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
676
677 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Fcmp)) {
678 // FIXME: optionally print.
679 return;
680 }
681
682 if (flags().enable_warning_stats)
683 nsan_stats->AddWarning(check_ty: CheckTypeT::kFcmp, pc, bp, rel_err: 0.0);
684
685 if (flags().disable_warnings || !flags().check_cmp)
686 return;
687
688 // FIXME: ideally we would print the shadow value as FP128. Right now because
689 // we truncate to long double we can sometimes see stuff like:
690 // shadow <value> == <value> (false)
691 using ValuePrinter = FTPrinter<FT>;
692 using ShadowPrinter = FTPrinter<ShadowFT>;
693 Decorator D;
694 const char *const PredicateName = GetPredicateName(v: Predicate);
695 Printf(format: "%s", D.Warning());
696 Printf(format: "WARNING: NumericalStabilitySanitizer: floating-point comparison "
697 "results depend on precision\n");
698 Printf(format: "%s", D.Default());
699 Printf("%-12s precision dec (native): %s %s %s (%s)\n"
700 "%-12s precision dec (shadow): %s %s %s (%s)\n"
701 "%-12s precision hex (native): %s %s %s (%s)\n"
702 "%-12s precision hex (shadow): %s %s %s (%s)\n"
703 "%s",
704 // Native, decimal.
705 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,
706 ValuePrinter::dec(Rhs).Buffer, GetTruthValueName(v: result),
707 // Shadow, decimal
708 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,
709 PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,
710 GetTruthValueName(v: ShadowResult),
711 // Native, hex.
712 FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,
713 ValuePrinter::hex(Rhs).Buffer, GetTruthValueName(v: result),
714 // Shadow, hex
715 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,
716 PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,
717 GetTruthValueName(v: ShadowResult), D.End());
718 stack.Print();
719 if (flags().halt_on_error) {
720 Printf(format: "Exiting\n");
721 Die();
722 }
723}
724
725extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
726__nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,
727 double rhs_shadow, int predicate, bool result,
728 bool shadow_result) {
729 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
730 ShadowResult: shadow_result);
731}
732
733extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
734__nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,
735 __float128 rhs_shadow, int predicate, bool result,
736 bool shadow_result) {
737 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
738 ShadowResult: shadow_result);
739}
740
741extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
742__nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,
743 long double rhs_shadow, int predicate, bool result,
744 bool shadow_result) {
745 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
746 ShadowResult: shadow_result);
747}
748
749extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
750__nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,
751 __float128 lhs_shadow, __float128 rhs_shadow,
752 int predicate, bool result, bool shadow_result) {
753 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
754 ShadowResult: shadow_result);
755}
756
757template <typename FT> void checkFTFromShadowStack(const FT value) {
758 // Get the shadow 2FT value from the shadow stack. Note that
759 // __nsan_check_{float,double,long double} is a function like any other, so
760 // the instrumentation will have placed the shadow value on the shadow stack.
761 using ShadowFT = typename FTInfo<FT>::shadow_type;
762 ShadowFT Shadow;
763 __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
764 checkFT(value, Shadow, CheckTypeT::kUser, 0);
765}
766
767// FIXME: Add suffixes and let the instrumentation pass automatically add
768// suffixes.
769extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float value) {
770 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&
771 "__nsan_check_float called from non-instrumented function");
772 checkFTFromShadowStack(value);
773}
774
775extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
776__nsan_check_double(double value) {
777 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&
778 "__nsan_check_double called from non-instrumented function");
779 checkFTFromShadowStack(value);
780}
781
782extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
783__nsan_check_longdouble(long double value) {
784 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&
785 "__nsan_check_longdouble called from non-instrumented function");
786 checkFTFromShadowStack(value);
787}
788
789template <typename FT> static void dumpFTFromShadowStack(const FT value) {
790 // Get the shadow 2FT value from the shadow stack. Note that
791 // __nsan_dump_{float,double,long double} is a function like any other, so
792 // the instrumentation will have placed the shadow value on the shadow stack.
793 using ShadowFT = typename FTInfo<FT>::shadow_type;
794 ShadowFT shadow;
795 __builtin_memcpy(&shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
796 using ValuePrinter = FTPrinter<FT>;
797 using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;
798 printf("value dec:%s hex:%s\n"
799 "shadow dec:%s hex:%s\n",
800 ValuePrinter::dec(value).Buffer, ValuePrinter::hex(value).Buffer,
801 ShadowPrinter::dec(shadow).Buffer, ShadowPrinter::hex(shadow).Buffer);
802}
803
804extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float value) {
805 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&
806 "__nsan_dump_float called from non-instrumented function");
807 dumpFTFromShadowStack(value);
808}
809
810extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double value) {
811 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&
812 "__nsan_dump_double called from non-instrumented function");
813 dumpFTFromShadowStack(value);
814}
815
816extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
817__nsan_dump_longdouble(long double value) {
818 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&
819 "__nsan_dump_longdouble called from non-instrumented function");
820 dumpFTFromShadowStack(value);
821}
822
823extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {
824 printf(format: "ret tag: %lx\n", __nsan_shadow_ret_tag);
825 double v;
826 __builtin_memcpy(&v, __nsan_shadow_ret_ptr, sizeof(double));
827 printf(format: "double value: %f\n", v);
828 // FIXME: float128 value.
829}
830
831extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
832 printf(format: "args tag: %lx\n", __nsan_shadow_args_tag);
833}
834
835bool __nsan::nsan_initialized;
836bool __nsan::nsan_init_is_running;
837
838extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
839 CHECK(!nsan_init_is_running);
840 if (nsan_initialized)
841 return;
842 nsan_init_is_running = true;
843 SanitizerToolName = "NumericalStabilitySanitizer";
844
845 InitializeFlags();
846 InitializeSuppressions();
847 InitializePlatformEarly();
848
849 DisableCoreDumperIfNecessary();
850
851 if (!MmapFixedNoReserve(fixed_addr: TypesAddr(), size: AllocatorAddr() - TypesAddr()))
852 Die();
853
854 InitializeInterceptors();
855 NsanTSDInit(destructor: NsanTSDDtor);
856 NsanAllocatorInit();
857
858 NsanThread *main_thread = NsanThread::Create(start_routine: nullptr, arg: nullptr);
859 SetCurrentThread(main_thread);
860 main_thread->Init();
861
862 InitializeStats();
863 if (flags().print_stats_on_exit)
864 Atexit(function: NsanAtexit);
865
866 nsan_init_is_running = false;
867 nsan_initialized = true;
868}
869