1//===-- nsan.cc -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// NumericalStabilitySanitizer runtime.
10//
11// This implements:
12// - The public nsan interface (include/sanitizer/nsan_interface.h).
13// - The private nsan interface (./nsan.h).
14// - The internal instrumentation interface. These are function emitted by the
15// instrumentation pass:
16// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load
17// These return the shadow memory pointer for loading the shadow value,
18// after checking that the types are consistent. If the types are not
19// consistent, returns nullptr.
20// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store
21// Sets the shadow types appropriately and returns the shadow memory
22// pointer for storing the shadow value.
23// * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the
24// accuracy of a value against its shadow and emits a warning depending
25// on the runtime configuration. The middle part indicates the type of
26// the application value, the suffix (f,d,l) indicates the type of the
27// shadow, and depends on the instrumentation configuration.
28// * __nsan_fcmp_fail_* emits a warning for a fcmp instruction whose
29// corresponding shadow fcmp result differs.
30//
31//===----------------------------------------------------------------------===//
32
33#include "nsan.h"
34#include "nsan_flags.h"
35#include "nsan_stats.h"
36#include "nsan_suppressions.h"
37#include "nsan_thread.h"
38
39#include <assert.h>
40#include <math.h>
41#include <stdint.h>
42#include <stdio.h>
43#include <stdlib.h>
44
45#include "sanitizer_common/sanitizer_atomic.h"
46#include "sanitizer_common/sanitizer_common.h"
47#include "sanitizer_common/sanitizer_libc.h"
48#include "sanitizer_common/sanitizer_report_decorator.h"
49#include "sanitizer_common/sanitizer_stacktrace.h"
50#include "sanitizer_common/sanitizer_symbolizer.h"
51
52using namespace __sanitizer;
53using namespace __nsan;
54
55constexpr int kMaxVectorWidth = 8;
56
57// When copying application memory, we also copy its shadow and shadow type.
58extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
59__nsan_copy_values(const void *daddr, const void *saddr, uptr size) {
60 internal_memmove(dest: GetShadowTypeAddrFor(ptr: daddr), src: GetShadowTypeAddrFor(ptr: saddr),
61 n: size);
62 internal_memmove(dest: GetShadowAddrFor(ptr: daddr), src: GetShadowAddrFor(ptr: saddr),
63 n: size * kShadowScale);
64}
65
66#define NSAN_COPY_VALUES_N(N) \
67 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_copy_##N( \
68 const u8 *daddr, const u8 *saddr) { \
69 __builtin_memmove(GetShadowTypeAddrFor(daddr), \
70 GetShadowTypeAddrFor(saddr), N); \
71 __builtin_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr), \
72 N *kShadowScale); \
73 }
74
75NSAN_COPY_VALUES_N(4)
76NSAN_COPY_VALUES_N(8)
77NSAN_COPY_VALUES_N(16)
78
79extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
80__nsan_set_value_unknown(const void *addr, uptr size) {
81 internal_memset(s: GetShadowTypeAddrFor(ptr: addr), c: 0, n: size);
82}
83
84#define NSAN_SET_VALUE_UNKNOWN_N(N) \
85 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_set_value_unknown_##N( \
86 const u8 *daddr) { \
87 __builtin_memset(GetShadowTypeAddrFor(daddr), 0, N); \
88 }
89
90NSAN_SET_VALUE_UNKNOWN_N(4)
91NSAN_SET_VALUE_UNKNOWN_N(8)
92NSAN_SET_VALUE_UNKNOWN_N(16)
93
94const char *FTInfo<float>::kCppTypeName = "float";
95const char *FTInfo<double>::kCppTypeName = "double";
96const char *FTInfo<long double>::kCppTypeName = "long double";
97const char *FTInfo<__float128>::kCppTypeName = "__float128";
98
99const char FTInfo<float>::kTypePattern[sizeof(float)];
100const char FTInfo<double>::kTypePattern[sizeof(double)];
101const char FTInfo<long double>::kTypePattern[sizeof(long double)];
102
103// Helper for __nsan_dump_shadow_mem: Reads the value at address `ptr`,
104// identified by its type id.
105template <typename ShadowFT>
106static __float128 ReadShadowInternal(const u8 *ptr) {
107 ShadowFT Shadow;
108 __builtin_memcpy(&Shadow, ptr, sizeof(Shadow));
109 return Shadow;
110}
111
112static __float128 ReadShadow(const u8 *ptr, const char ShadowTypeId) {
113 switch (ShadowTypeId) {
114 case 'd':
115 return ReadShadowInternal<double>(ptr);
116 case 'l':
117 return ReadShadowInternal<long double>(ptr);
118 case 'q':
119 return ReadShadowInternal<__float128>(ptr);
120 default:
121 return 0.0;
122 }
123}
124
125namespace {
126class Decorator : public __sanitizer::SanitizerCommonDecorator {
127public:
128 Decorator() : SanitizerCommonDecorator() {}
129 const char *Warning() { return Red(); }
130 const char *Name() { return Green(); }
131 const char *End() { return Default(); }
132};
133
134// Workaround for the fact that Printf() does not support floats.
135struct PrintBuffer {
136 char Buffer[64];
137};
138template <typename FT> struct FTPrinter {};
139
140template <> struct FTPrinter<double> {
141 static PrintBuffer dec(double value) {
142 PrintBuffer result;
143 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20f", value);
144 return result;
145 }
146 static PrintBuffer hex(double value) {
147 PrintBuffer result;
148 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20a", value);
149 return result;
150 }
151};
152
153template <> struct FTPrinter<float> : FTPrinter<double> {};
154
155template <> struct FTPrinter<long double> {
156 static PrintBuffer dec(long double value) {
157 PrintBuffer result;
158 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20Lf", value);
159 return result;
160 }
161 static PrintBuffer hex(long double value) {
162 PrintBuffer result;
163 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20La", value);
164 return result;
165 }
166};
167
168// FIXME: print with full precision.
169template <> struct FTPrinter<__float128> : FTPrinter<long double> {};
170
171// This is a template so that there are no implicit conversions.
172template <typename FT> inline FT ftAbs(FT v);
173
174template <> inline long double ftAbs(long double v) { return fabsl(x: v); }
175template <> inline double ftAbs(double v) { return fabs(x: v); }
176
177// We don't care about nans.
178// std::abs(__float128) code is suboptimal and generates a function call to
179// __getf2().
180template <typename FT> inline FT ftAbs(FT v) { return v >= FT{0} ? v : -v; }
181
182template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {
183 using type = FT2;
184};
185
186template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {
187 using type = FT1;
188};
189
190template <typename FT1, typename FT2>
191using LargestFT =
192 typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;
193
194template <typename T> T max(T a, T b) { return a < b ? b : a; }
195
196} // end anonymous namespace
197
198void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
199 void *context,
200 bool request_fast,
201 u32 max_depth) {
202 using namespace __nsan;
203 NsanThread *t = GetCurrentThread();
204 if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast))
205 return Unwind(max_depth, pc, bp, context, stack_top: t ? t->stack_top() : 0,
206 stack_bottom: t ? t->stack_bottom() : 0, request_fast_unwind: false);
207 if (StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast))
208 Unwind(max_depth, pc, bp, context: nullptr, stack_top: t->stack_top(), stack_bottom: t->stack_bottom(), request_fast_unwind: true);
209 else
210 Unwind(max_depth, pc, bp: 0, context, stack_top: 0, stack_bottom: 0, request_fast_unwind: false);
211}
212
213extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {
214 if (nsan_stats)
215 nsan_stats->Print();
216}
217
218static void NsanAtexit() {
219 Printf(format: "Numerical Sanitizer exit stats:\n");
220 __nsan_print_accumulated_stats();
221 nsan_stats = nullptr;
222}
223
224// The next three functions return a pointer for storing a shadow value for `n`
225// values, after setting the shadow types. We return the pointer instead of
226// storing ourselves because it avoids having to rely on the calling convention
227// around long double being the same for nsan and the target application.
228// We have to have 3 versions because we need to know which type we are storing
229// since we are setting the type shadow memory.
230template <typename FT> static u8 *getShadowPtrForStore(u8 *store_addr, uptr n) {
231 unsigned char *shadow_type = GetShadowTypeAddrFor(ptr: store_addr);
232 for (uptr i = 0; i < n; ++i) {
233 __builtin_memcpy(shadow_type + i * sizeof(FT), FTInfo<FT>::kTypePattern,
234 sizeof(FTInfo<FT>::kTypePattern));
235 }
236 return GetShadowAddrFor(ptr: store_addr);
237}
238
239extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
240__nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {
241 return getShadowPtrForStore<float>(store_addr, n);
242}
243
244extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
245__nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {
246 return getShadowPtrForStore<double>(store_addr, n);
247}
248
249extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
250__nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {
251 return getShadowPtrForStore<long double>(store_addr, n);
252}
253
254template <typename FT> static bool IsValidShadowType(const u8 *shadow_type) {
255 return __builtin_memcmp(shadow_type, FTInfo<FT>::kTypePattern, sizeof(FT)) ==
256 0;
257}
258
259template <int kSize, typename T> static bool IsZero(const T *ptr) {
260 constexpr const char kZeros[kSize] = {}; // Zero initialized.
261 return __builtin_memcmp(ptr, kZeros, kSize) == 0;
262}
263
264template <typename FT> static bool IsUnknownShadowType(const u8 *shadow_type) {
265 return IsZero<sizeof(FTInfo<FT>::kTypePattern)>(shadow_type);
266}
267
268// The three folowing functions check that the address stores a complete
269// shadow value of the given type and return a pointer for loading.
270// They return nullptr if the type of the value is unknown or incomplete.
271template <typename FT>
272static const u8 *getShadowPtrForLoad(const u8 *load_addr, uptr n) {
273 const u8 *const shadow_type = GetShadowTypeAddrFor(ptr: load_addr);
274 for (uptr i = 0; i < n; ++i) {
275 if (!IsValidShadowType<FT>(shadow_type + i * sizeof(FT))) {
276 // If loadtracking stats are enabled, log loads with invalid types
277 // (tampered with through type punning).
278 if (flags().enable_loadtracking_stats) {
279 if (IsUnknownShadowType<FT>(shadow_type + i * sizeof(FT))) {
280 // Warn only if the value is non-zero. Zero is special because
281 // applications typically initialize large buffers to zero in an
282 // untyped way.
283 if (!IsZero<sizeof(FT)>(load_addr)) {
284 GET_CALLER_PC_BP;
285 nsan_stats->AddUnknownLoadTrackingEvent(pc, bp);
286 }
287 } else {
288 GET_CALLER_PC_BP;
289 nsan_stats->AddInvalidLoadTrackingEvent(pc, bp);
290 }
291 }
292 return nullptr;
293 }
294 }
295 return GetShadowAddrFor(ptr: load_addr);
296}
297
298extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
299__nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {
300 return getShadowPtrForLoad<float>(load_addr, n);
301}
302
303extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
304__nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {
305 return getShadowPtrForLoad<double>(load_addr, n);
306}
307
308extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
309__nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {
310 return getShadowPtrForLoad<long double>(load_addr, n);
311}
312
313// Returns the raw shadow pointer. The returned pointer should be considered
314// opaque.
315extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
316__nsan_internal_get_raw_shadow_ptr(const u8 *addr) {
317 return GetShadowAddrFor(ptr: addr);
318}
319
320// Returns the raw shadow type pointer. The returned pointer should be
321// considered opaque.
322extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
323__nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {
324 return reinterpret_cast<u8 *>(GetShadowTypeAddrFor(ptr: addr));
325}
326
327static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }
328
329static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }
330
331// The instrumentation automatically appends `shadow_value_type_ids`, see
332// maybeAddSuffixForNsanInterface.
333extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
334__nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,
335 size_t shadow_value_type_ids) {
336 const u8 *const shadow_type = GetShadowTypeAddrFor(ptr: addr);
337 const u8 *const shadow = GetShadowAddrFor(ptr: addr);
338
339 constexpr int kMaxNumDecodedValues = 16;
340 __float128 decoded_values[kMaxNumDecodedValues];
341 int num_decoded_values = 0;
342 if (bytes_per_line > 4 * kMaxNumDecodedValues)
343 bytes_per_line = 4 * kMaxNumDecodedValues;
344
345 // We keep track of the current type and position as we go.
346 ValueType LastValueTy = kUnknownValueType;
347 int LastPos = -1;
348 size_t Offset = 0;
349 for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;
350 ++R) {
351 printf(format: "%p: ", (void *)(addr + R * bytes_per_line));
352 for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {
353 const ValueType ValueTy = getValueType(c: shadow_type[Offset]);
354 const int pos = getValuePos(c: shadow_type[Offset]);
355 if (ValueTy == LastValueTy && pos == LastPos + 1) {
356 ++LastPos;
357 } else {
358 LastValueTy = ValueTy;
359 LastPos = pos == 0 ? 0 : -1;
360 }
361
362 switch (ValueTy) {
363 case kUnknownValueType:
364 printf(format: "__ ");
365 break;
366 case kFloatValueType:
367 printf(format: "f%x ", pos);
368 if (LastPos == sizeof(float) - 1) {
369 decoded_values[num_decoded_values] =
370 ReadShadow(ptr: shadow + kShadowScale * (Offset + 1 - sizeof(float)),
371 ShadowTypeId: static_cast<char>(shadow_value_type_ids & 0xff));
372 ++num_decoded_values;
373 }
374 break;
375 case kDoubleValueType:
376 printf(format: "d%x ", pos);
377 if (LastPos == sizeof(double) - 1) {
378 decoded_values[num_decoded_values] = ReadShadow(
379 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(double)),
380 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 8) & 0xff));
381 ++num_decoded_values;
382 }
383 break;
384 case kFp80ValueType:
385 printf(format: "l%x ", pos);
386 if (LastPos == sizeof(long double) - 1) {
387 decoded_values[num_decoded_values] = ReadShadow(
388 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(long double)),
389 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 16) & 0xff));
390 ++num_decoded_values;
391 }
392 break;
393 }
394 ++Offset;
395 }
396 for (int i = 0; i < num_decoded_values; ++i) {
397 printf(format: " (%s)", FTPrinter<__float128>::dec(value: decoded_values[i]).Buffer);
398 }
399 num_decoded_values = 0;
400 printf(format: "\n");
401 }
402}
403
404alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
405 thread_local uptr __nsan_shadow_ret_tag = 0;
406
407alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
408 thread_local char __nsan_shadow_ret_ptr[kMaxVectorWidth *
409 sizeof(__float128)];
410
411alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
412 thread_local uptr __nsan_shadow_args_tag = 0;
413
414// Maximum number of args. This should be enough for anyone (tm). An alternate
415// scheme is to have the generated code create an alloca and make
416// __nsan_shadow_args_ptr point ot the alloca.
417constexpr const int kMaxNumArgs = 128;
418alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
419 thread_local char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs *
420 sizeof(__float128)];
421
422enum ContinuationType { // Keep in sync with instrumentation pass.
423 kContinueWithShadow = 0,
424 kResumeFromValue = 1,
425};
426
427// Checks the consistency between application and shadow value. Returns true
428// when the instrumented code should resume computations from the original value
429// rather than the shadow value. This prevents one error to propagate to all
430// subsequent operations. This behaviour is tunable with flags.
431template <typename FT, typename ShadowFT>
432int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,
433 uptr CheckArg) {
434 // We do all comparisons in the InternalFT domain, which is the largest FT
435 // type.
436 using InternalFT = LargestFT<FT, ShadowFT>;
437 const InternalFT check_value = value;
438 const InternalFT check_shadow = Shadow;
439
440 // We only check for NaNs in the value, not the shadow.
441 if (flags().check_nan && isnan(value)) {
442 GET_CALLER_PC_BP;
443 BufferedStackTrace stack;
444 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
445 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Consistency)) {
446 // FIXME: optionally print.
447 return flags().resume_after_suppression ? kResumeFromValue
448 : kContinueWithShadow;
449 }
450 Decorator D;
451 Printf(format: "%s", D.Warning());
452 Printf(format: "WARNING: NumericalStabilitySanitizer: NaN detected\n");
453 Printf(format: "%s", D.Default());
454 stack.Print();
455 if (flags().halt_on_error) {
456 if (common_flags()->abort_on_error)
457 Printf(format: "ABORTING\n");
458 else
459 Printf(format: "Exiting\n");
460 Die();
461 }
462 // Performing other tests for NaN values is meaningless when dealing with numbers.
463 return kResumeFromValue;
464 }
465
466 // See this article for an interesting discussion of how to compare floats:
467 // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
468 static constexpr const FT Eps = FTInfo<FT>::kEpsilon;
469
470 const InternalFT abs_err = ftAbs(check_value - check_shadow);
471
472 if (flags().enable_check_stats) {
473 GET_CALLER_PC_BP;
474 // We are re-computing `largest` here because this is a cold branch, and we
475 // want to avoid having to move the computation of `largest` before the
476 // absolute value check when this branch is not taken.
477 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
478 nsan_stats->AddCheck(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
479 }
480
481 // Note: writing the comparison that way ensures that when `abs_err` is Nan
482 // (value and shadow are inf or -inf), we pass the test.
483 if (!(abs_err >= flags().cached_absolute_error_threshold))
484 return kContinueWithShadow;
485
486 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
487 if (abs_err * (1ull << flags().log2_max_relative_error) <= largest)
488 return kContinueWithShadow; // No problem here.
489
490 if (!flags().disable_warnings) {
491 GET_CALLER_PC_BP;
492 UNINITIALIZED BufferedStackTrace stack;
493 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
494 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Consistency)) {
495 // FIXME: optionally print.
496 return flags().resume_after_suppression ? kResumeFromValue
497 : kContinueWithShadow;
498 }
499
500 Decorator D;
501 Printf(format: "%s", D.Warning());
502 // Printf does not support float formatting.
503 char RelErrBuf[64] = "inf";
504 if (largest > Eps) {
505 snprintf(s: RelErrBuf, maxlen: sizeof(RelErrBuf) - 1, format: "%.20Lf%% (2^%.0Lf epsilons)",
506 static_cast<long double>(100.0 * abs_err / largest),
507 log2l(x: static_cast<long double>(abs_err / largest / Eps)));
508 }
509 char ulp_err_buf[128] = "";
510 const double shadow_ulp_diff = GetULPDiff(check_value, check_shadow);
511 if (shadow_ulp_diff != kMaxULPDiff) {
512 // This is the ULP diff in the internal domain. The user actually cares
513 // about that in the original domain.
514 const double ulp_diff =
515 shadow_ulp_diff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -
516 FTInfo<FT>::kMantissaBits));
517 snprintf(s: ulp_err_buf, maxlen: sizeof(ulp_err_buf) - 1,
518 format: "(%.0f ULPs == %.1f digits == %.1f bits)", ulp_diff,
519 log10(x: ulp_diff), log2(x: ulp_diff));
520 }
521 Printf(format: "WARNING: NumericalStabilitySanitizer: inconsistent shadow results");
522 switch (CheckType) {
523 case CheckTypeT::kUnknown:
524 case CheckTypeT::kFcmp:
525 case CheckTypeT::kMaxCheckType:
526 break;
527 case CheckTypeT::kRet:
528 Printf(format: " while checking return value");
529 break;
530 case CheckTypeT::kArg:
531 Printf(format: " while checking call argument #%d", static_cast<int>(CheckArg));
532 break;
533 case CheckTypeT::kLoad:
534 Printf(
535 format: " while checking load from address 0x%lx. This is due to incorrect "
536 "shadow memory tracking, typically due to uninstrumented code "
537 "writing to memory.",
538 CheckArg);
539 break;
540 case CheckTypeT::kStore:
541 Printf(format: " while checking store to address 0x%lx", CheckArg);
542 break;
543 case CheckTypeT::kInsert:
544 Printf(format: " while checking vector insert");
545 break;
546 case CheckTypeT::kUser:
547 Printf(format: " in user-initiated check");
548 break;
549 }
550 using ValuePrinter = FTPrinter<FT>;
551 using ShadowPrinter = FTPrinter<ShadowFT>;
552 Printf(format: "%s", D.Default());
553
554 Printf("\n"
555 "%-12s precision (native): dec: %s hex: %s\n"
556 "%-12s precision (shadow): dec: %s hex: %s\n"
557 "shadow truncated to %-12s: dec: %s hex: %s\n"
558 "Relative error: %s\n"
559 "Absolute error: %s\n"
560 "%s\n",
561 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(value).Buffer,
562 ValuePrinter::hex(value).Buffer, FTInfo<ShadowFT>::kCppTypeName,
563 ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,
564 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,
565 ValuePrinter::hex(Shadow).Buffer, RelErrBuf,
566 ValuePrinter::hex(abs_err).Buffer, ulp_err_buf);
567 stack.Print();
568 }
569
570 if (flags().enable_warning_stats) {
571 GET_CALLER_PC_BP;
572 nsan_stats->AddWarning(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
573 }
574
575 if (flags().halt_on_error) {
576 if (common_flags()->abort_on_error)
577 Printf(format: "ABORTING\n");
578 else
579 Printf(format: "Exiting\n");
580 Die();
581 }
582 return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
583}
584
585extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(
586 float value, double shadow, int32_t check_type, uptr check_arg) {
587 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
588}
589
590extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(
591 double value, long double shadow, int32_t check_type, uptr check_arg) {
592 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
593}
594
595extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(
596 double value, __float128 shadow, int32_t check_type, uptr check_arg) {
597 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
598}
599
600extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t
601__nsan_internal_check_longdouble_q(long double value, __float128 shadow,
602 int32_t check_type, uptr check_arg) {
603 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
604}
605
606static const char *GetTruthValueName(bool v) { return v ? "true" : "false"; }
607
608// This uses the same values as CmpInst::Predicate.
609static const char *GetPredicateName(int v) {
610 switch (v) {
611 case 0:
612 return "(false)";
613 case 1:
614 return "==";
615 case 2:
616 return ">";
617 case 3:
618 return ">=";
619 case 4:
620 return "<";
621 case 5:
622 return "<=";
623 case 6:
624 return "!=";
625 case 7:
626 return "(ordered)";
627 case 8:
628 return "(unordered)";
629 case 9:
630 return "==";
631 case 10:
632 return ">";
633 case 11:
634 return ">=";
635 case 12:
636 return "<";
637 case 13:
638 return "<=";
639 case 14:
640 return "!=";
641 case 15:
642 return "(true)";
643 }
644 return "??";
645}
646
647template <typename FT, typename ShadowFT>
648void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,
649 ShadowFT RhsShadow, int Predicate, bool result,
650 bool ShadowResult) {
651 if (result == ShadowResult) {
652 // When a vector comparison fails, we fail each element of the comparison
653 // to simplify instrumented code. Skip elements where the shadow comparison
654 // gave the same result as the original one.
655 return;
656 }
657
658 GET_CALLER_PC_BP;
659 UNINITIALIZED BufferedStackTrace stack;
660 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
661
662 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Fcmp)) {
663 // FIXME: optionally print.
664 return;
665 }
666
667 if (flags().enable_warning_stats)
668 nsan_stats->AddWarning(check_ty: CheckTypeT::kFcmp, pc, bp, rel_err: 0.0);
669
670 if (flags().disable_warnings || !flags().check_cmp)
671 return;
672
673 // FIXME: ideally we would print the shadow value as FP128. Right now because
674 // we truncate to long double we can sometimes see stuff like:
675 // shadow <value> == <value> (false)
676 using ValuePrinter = FTPrinter<FT>;
677 using ShadowPrinter = FTPrinter<ShadowFT>;
678 Decorator D;
679 const char *const PredicateName = GetPredicateName(v: Predicate);
680 Printf(format: "%s", D.Warning());
681 Printf(format: "WARNING: NumericalStabilitySanitizer: floating-point comparison "
682 "results depend on precision\n");
683 Printf(format: "%s", D.Default());
684 Printf("%-12s precision dec (native): %s %s %s (%s)\n"
685 "%-12s precision dec (shadow): %s %s %s (%s)\n"
686 "%-12s precision hex (native): %s %s %s (%s)\n"
687 "%-12s precision hex (shadow): %s %s %s (%s)\n"
688 "%s",
689 // Native, decimal.
690 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,
691 ValuePrinter::dec(Rhs).Buffer, GetTruthValueName(v: result),
692 // Shadow, decimal
693 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,
694 PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,
695 GetTruthValueName(v: ShadowResult),
696 // Native, hex.
697 FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,
698 ValuePrinter::hex(Rhs).Buffer, GetTruthValueName(v: result),
699 // Shadow, hex
700 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,
701 PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,
702 GetTruthValueName(v: ShadowResult), D.End());
703 stack.Print();
704 if (flags().halt_on_error) {
705 Printf(format: "Exiting\n");
706 Die();
707 }
708}
709
710extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
711__nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,
712 double rhs_shadow, int predicate, bool result,
713 bool shadow_result) {
714 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
715 ShadowResult: shadow_result);
716}
717
718extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
719__nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,
720 __float128 rhs_shadow, int predicate, bool result,
721 bool shadow_result) {
722 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
723 ShadowResult: shadow_result);
724}
725
726extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
727__nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,
728 long double rhs_shadow, int predicate, bool result,
729 bool shadow_result) {
730 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
731 ShadowResult: shadow_result);
732}
733
734extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
735__nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,
736 __float128 lhs_shadow, __float128 rhs_shadow,
737 int predicate, bool result, bool shadow_result) {
738 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
739 ShadowResult: shadow_result);
740}
741
742template <typename FT> void checkFTFromShadowStack(const FT value) {
743 // Get the shadow 2FT value from the shadow stack. Note that
744 // __nsan_check_{float,double,long double} is a function like any other, so
745 // the instrumentation will have placed the shadow value on the shadow stack.
746 using ShadowFT = typename FTInfo<FT>::shadow_type;
747 ShadowFT Shadow;
748 __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
749 checkFT(value, Shadow, CheckTypeT::kUser, 0);
750}
751
752// FIXME: Add suffixes and let the instrumentation pass automatically add
753// suffixes.
754extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float value) {
755 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&
756 "__nsan_check_float called from non-instrumented function");
757 checkFTFromShadowStack(value);
758}
759
760extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
761__nsan_check_double(double value) {
762 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&
763 "__nsan_check_double called from non-instrumented function");
764 checkFTFromShadowStack(value);
765}
766
767extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
768__nsan_check_longdouble(long double value) {
769 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&
770 "__nsan_check_longdouble called from non-instrumented function");
771 checkFTFromShadowStack(value);
772}
773
774template <typename FT> static void dumpFTFromShadowStack(const FT value) {
775 // Get the shadow 2FT value from the shadow stack. Note that
776 // __nsan_dump_{float,double,long double} is a function like any other, so
777 // the instrumentation will have placed the shadow value on the shadow stack.
778 using ShadowFT = typename FTInfo<FT>::shadow_type;
779 ShadowFT shadow;
780 __builtin_memcpy(&shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
781 using ValuePrinter = FTPrinter<FT>;
782 using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;
783 printf("value dec:%s hex:%s\n"
784 "shadow dec:%s hex:%s\n",
785 ValuePrinter::dec(value).Buffer, ValuePrinter::hex(value).Buffer,
786 ShadowPrinter::dec(shadow).Buffer, ShadowPrinter::hex(shadow).Buffer);
787}
788
789extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float value) {
790 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&
791 "__nsan_dump_float called from non-instrumented function");
792 dumpFTFromShadowStack(value);
793}
794
795extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double value) {
796 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&
797 "__nsan_dump_double called from non-instrumented function");
798 dumpFTFromShadowStack(value);
799}
800
801extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
802__nsan_dump_longdouble(long double value) {
803 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&
804 "__nsan_dump_longdouble called from non-instrumented function");
805 dumpFTFromShadowStack(value);
806}
807
808extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {
809 printf(format: "ret tag: %lx\n", __nsan_shadow_ret_tag);
810 double v;
811 __builtin_memcpy(&v, __nsan_shadow_ret_ptr, sizeof(double));
812 printf(format: "double value: %f\n", v);
813 // FIXME: float128 value.
814}
815
816extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
817 printf(format: "args tag: %lx\n", __nsan_shadow_args_tag);
818}
819
820bool __nsan::nsan_initialized;
821bool __nsan::nsan_init_is_running;
822
823extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
824 CHECK(!nsan_init_is_running);
825 if (nsan_initialized)
826 return;
827 nsan_init_is_running = true;
828 SanitizerToolName = "NumericalStabilitySanitizer";
829
830 InitializeFlags();
831 InitializeSuppressions();
832 InitializePlatformEarly();
833
834 DisableCoreDumperIfNecessary();
835
836 if (!MmapFixedNoReserve(fixed_addr: TypesAddr(), size: AllocatorAddr() - TypesAddr()))
837 Die();
838
839 InitializeInterceptors();
840 NsanTSDInit(destructor: NsanTSDDtor);
841 NsanAllocatorInit();
842
843 NsanThread *main_thread = NsanThread::Create(start_routine: nullptr, arg: nullptr);
844 SetCurrentThread(main_thread);
845 main_thread->Init();
846
847 InitializeStats();
848 if (flags().print_stats_on_exit)
849 Atexit(function: NsanAtexit);
850
851 nsan_init_is_running = false;
852 nsan_initialized = true;
853}
854