1//===-- nsan.cc -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// NumericalStabilitySanitizer runtime.
10//
11// This implements:
12// - The public nsan interface (include/sanitizer/nsan_interface.h).
13// - The private nsan interface (./nsan.h).
14// - The internal instrumentation interface. These are function emitted by the
15// instrumentation pass:
16// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load
17// These return the shadow memory pointer for loading the shadow value,
18// after checking that the types are consistent. If the types are not
19// consistent, returns nullptr.
20// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store
21// Sets the shadow types appropriately and returns the shadow memory
22// pointer for storing the shadow value.
23// * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the
24// accuracy of a value against its shadow and emits a warning depending
25// on the runtime configuration. The middle part indicates the type of
26// the application value, the suffix (f,d,l) indicates the type of the
27// shadow, and depends on the instrumentation configuration.
28// * __nsan_fcmp_fail_* emits a warning for an fcmp instruction whose
29// corresponding shadow fcmp result differs.
30//
31//===----------------------------------------------------------------------===//
32
33#include <assert.h>
34#include <math.h>
35#include <stdint.h>
36#include <stdio.h>
37#include <stdlib.h>
38
39#include "sanitizer_common/sanitizer_atomic.h"
40#include "sanitizer_common/sanitizer_common.h"
41#include "sanitizer_common/sanitizer_libc.h"
42#include "sanitizer_common/sanitizer_report_decorator.h"
43#include "sanitizer_common/sanitizer_stacktrace.h"
44#include "sanitizer_common/sanitizer_symbolizer.h"
45
46#include "nsan/nsan.h"
47#include "nsan/nsan_flags.h"
48#include "nsan/nsan_stats.h"
49#include "nsan/nsan_suppressions.h"
50
51using namespace __sanitizer;
52using namespace __nsan;
53
54constexpr int kMaxVectorWidth = 8;
55
56// When copying application memory, we also copy its shadow and shadow type.
57// FIXME: We could provide fixed-size versions that would nicely
58// vectorize for known sizes.
59extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
60__nsan_copy_values(const u8 *daddr, const u8 *saddr, uptr size) {
61 internal_memmove(dest: (void *)GetShadowTypeAddrFor(Ptr: daddr),
62 src: GetShadowTypeAddrFor(Ptr: saddr), n: size);
63 internal_memmove(dest: (void *)GetShadowAddrFor(Ptr: daddr), src: GetShadowAddrFor(Ptr: saddr),
64 n: size * kShadowScale);
65}
66
67// FIXME: We could provide fixed-size versions that would nicely
68// vectorize for known sizes.
69extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
70__nsan_set_value_unknown(const u8 *addr, uptr size) {
71 internal_memset(s: (void *)GetShadowTypeAddrFor(Ptr: addr), c: 0, n: size);
72}
73
74
75const char *FTInfo<float>::kCppTypeName = "float";
76const char *FTInfo<double>::kCppTypeName = "double";
77const char *FTInfo<long double>::kCppTypeName = "long double";
78const char *FTInfo<__float128>::kCppTypeName = "__float128";
79
80const char FTInfo<float>::kTypePattern[sizeof(float)];
81const char FTInfo<double>::kTypePattern[sizeof(double)];
82const char FTInfo<long double>::kTypePattern[sizeof(long double)];
83
84// Helper for __nsan_dump_shadow_mem: Reads the value at address `ptr`,
85// identified by its type id.
86template <typename ShadowFT>
87static __float128 ReadShadowInternal(const u8 *ptr) {
88 ShadowFT Shadow;
89 __builtin_memcpy(&Shadow, ptr, sizeof(Shadow));
90 return Shadow;
91}
92
93static __float128 ReadShadow(const u8 *ptr, const char ShadowTypeId) {
94 switch (ShadowTypeId) {
95 case 'd':
96 return ReadShadowInternal<double>(ptr);
97 case 'l':
98 return ReadShadowInternal<long double>(ptr);
99 case 'q':
100 return ReadShadowInternal<__float128>(ptr);
101 default:
102 return 0.0;
103 }
104}
105
106namespace {
107class Decorator : public __sanitizer::SanitizerCommonDecorator {
108public:
109 Decorator() : SanitizerCommonDecorator() {}
110 const char *Warning() { return Red(); }
111 const char *Name() { return Green(); }
112 const char *End() { return Default(); }
113};
114
115// Workaround for the fact that Printf() does not support floats.
116struct PrintBuffer {
117 char Buffer[64];
118};
119template <typename FT> struct FTPrinter {};
120
121template <> struct FTPrinter<double> {
122 static PrintBuffer dec(double value) {
123 PrintBuffer result;
124 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20f", value);
125 return result;
126 }
127 static PrintBuffer hex(double value) {
128 PrintBuffer result;
129 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20a", value);
130 return result;
131 }
132};
133
134template <> struct FTPrinter<float> : FTPrinter<double> {};
135
136template <> struct FTPrinter<long double> {
137 static PrintBuffer dec(long double value) {
138 PrintBuffer result;
139 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20Lf", value);
140 return result;
141 }
142 static PrintBuffer hex(long double value) {
143 PrintBuffer result;
144 snprintf(s: result.Buffer, maxlen: sizeof(result.Buffer) - 1, format: "%.20La", value);
145 return result;
146 }
147};
148
149// FIXME: print with full precision.
150template <> struct FTPrinter<__float128> : FTPrinter<long double> {};
151
152// This is a template so that there are no implicit conversions.
153template <typename FT> inline FT ftAbs(FT v);
154
155template <> inline long double ftAbs(long double v) { return fabsl(x: v); }
156template <> inline double ftAbs(double v) { return fabs(x: v); }
157
158// We don't care about nans.
159// std::abs(__float128) code is suboptimal and generates a function call to
160// __getf2().
161template <typename FT> inline FT ftAbs(FT v) { return v >= FT{0} ? v : -v; }
162
163template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {
164 using type = FT2;
165};
166
167template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {
168 using type = FT1;
169};
170
171template <typename FT1, typename FT2>
172using LargestFT =
173 typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;
174
175template <typename T> T max(T a, T b) { return a < b ? b : a; }
176
177} // end anonymous namespace
178
179void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
180 void *context,
181 bool request_fast,
182 u32 max_depth) {
183 using namespace __nsan;
184 return Unwind(max_depth, pc, bp, context, stack_top: 0, stack_bottom: 0, request_fast_unwind: false);
185}
186
187extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {
188 if (nsan_stats)
189 nsan_stats->Print();
190}
191
192static void NsanAtexit() {
193 Printf(format: "Numerical Sanitizer exit stats:\n");
194 __nsan_print_accumulated_stats();
195 nsan_stats = nullptr;
196}
197
198// The next three functions return a pointer for storing a shadow value for `n`
199// values, after setting the shadow types. We return the pointer instead of
200// storing ourselves because it avoids having to rely on the calling convention
201// around long double being the same for nsan and the target application.
202// We have to have 3 versions because we need to know which type we are storing
203// since we are setting the type shadow memory.
204template <typename FT> static u8 *getShadowPtrForStore(u8 *store_addr, uptr n) {
205 unsigned char *shadow_type = GetShadowTypeAddrFor(Ptr: store_addr);
206 for (uptr i = 0; i < n; ++i) {
207 __builtin_memcpy(shadow_type + i * sizeof(FT), FTInfo<FT>::kTypePattern,
208 sizeof(FTInfo<FT>::kTypePattern));
209 }
210 return GetShadowAddrFor(Ptr: store_addr);
211}
212
213extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
214__nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {
215 return getShadowPtrForStore<float>(store_addr, n);
216}
217
218extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
219__nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {
220 return getShadowPtrForStore<double>(store_addr, n);
221}
222
223extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
224__nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {
225 return getShadowPtrForStore<long double>(store_addr, n);
226}
227
228template <typename FT> static bool IsValidShadowType(const u8 *shadow_type) {
229 return __builtin_memcmp(shadow_type, FTInfo<FT>::kTypePattern, sizeof(FT)) ==
230 0;
231}
232
233template <int kSize, typename T> static bool IsZero(const T *ptr) {
234 constexpr const char kZeros[kSize] = {}; // Zero initialized.
235 return __builtin_memcmp(ptr, kZeros, kSize) == 0;
236}
237
238template <typename FT> static bool IsUnknownShadowType(const u8 *shadow_type) {
239 return IsZero<sizeof(FTInfo<FT>::kTypePattern)>(shadow_type);
240}
241
242// The three folowing functions check that the address stores a complete
243// shadow value of the given type and return a pointer for loading.
244// They return nullptr if the type of the value is unknown or incomplete.
245template <typename FT>
246static const u8 *getShadowPtrForLoad(const u8 *load_addr, uptr n) {
247 const u8 *const shadow_type = GetShadowTypeAddrFor(Ptr: load_addr);
248 for (uptr i = 0; i < n; ++i) {
249 if (!IsValidShadowType<FT>(shadow_type + i * sizeof(FT))) {
250 // If loadtracking stats are enabled, log loads with invalid types
251 // (tampered with through type punning).
252 if (flags().enable_loadtracking_stats) {
253 if (IsUnknownShadowType<FT>(shadow_type + i * sizeof(FT))) {
254 // Warn only if the value is non-zero. Zero is special because
255 // applications typically initialize large buffers to zero in an
256 // untyped way.
257 if (!IsZero<sizeof(FT)>(load_addr)) {
258 GET_CALLER_PC_BP;
259 nsan_stats->AddUnknownLoadTrackingEvent(pc, bp);
260 }
261 } else {
262 GET_CALLER_PC_BP;
263 nsan_stats->AddInvalidLoadTrackingEvent(pc, bp);
264 }
265 }
266 return nullptr;
267 }
268 }
269 return GetShadowAddrFor(Ptr: load_addr);
270}
271
272extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
273__nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {
274 return getShadowPtrForLoad<float>(load_addr, n);
275}
276
277extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
278__nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {
279 return getShadowPtrForLoad<double>(load_addr, n);
280}
281
282extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
283__nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {
284 return getShadowPtrForLoad<long double>(load_addr, n);
285}
286
287// Returns the raw shadow pointer. The returned pointer should be considered
288// opaque.
289extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
290__nsan_internal_get_raw_shadow_ptr(const u8 *addr) {
291 return GetShadowAddrFor(Ptr: const_cast<u8 *>(addr));
292}
293
294// Returns the raw shadow type pointer. The returned pointer should be
295// considered opaque.
296extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
297__nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {
298 return reinterpret_cast<u8 *>(GetShadowTypeAddrFor(Ptr: const_cast<u8 *>(addr)));
299}
300
301static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }
302
303static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }
304
305// Checks the consistency of the value types at the given type pointer.
306// If the value is inconsistent, returns ValueType::kUnknown. Else, return the
307// consistent type.
308template <typename FT>
309static bool checkValueConsistency(const u8 *shadow_type) {
310 const int pos = getValuePos(c: *shadow_type);
311 // Check that all bytes from the start of the value are ordered.
312 for (uptr i = 0; i < sizeof(FT); ++i) {
313 const u8 T = *(shadow_type - pos + i);
314 if (!(getValueType(c: T) == FTInfo<FT>::kValueType && getValuePos(c: T) == i))
315 return false;
316 }
317 return true;
318}
319
320// The instrumentation automatically appends `shadow_value_type_ids`, see
321// maybeAddSuffixForNsanInterface.
322extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
323__nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,
324 size_t shadow_value_type_ids) {
325 const u8 *const shadow_type = GetShadowTypeAddrFor(Ptr: addr);
326 const u8 *const shadow = GetShadowAddrFor(Ptr: addr);
327
328 constexpr int kMaxNumDecodedValues = 16;
329 __float128 decoded_values[kMaxNumDecodedValues];
330 int num_decoded_values = 0;
331 if (bytes_per_line > 4 * kMaxNumDecodedValues)
332 bytes_per_line = 4 * kMaxNumDecodedValues;
333
334 // We keep track of the current type and position as we go.
335 ValueType LastValueTy = kUnknownValueType;
336 int LastPos = -1;
337 size_t Offset = 0;
338 for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;
339 ++R) {
340 printf(format: "%p: ", (void *)(addr + R * bytes_per_line));
341 for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {
342 const ValueType ValueTy = getValueType(c: shadow_type[Offset]);
343 const int pos = getValuePos(c: shadow_type[Offset]);
344 if (ValueTy == LastValueTy && pos == LastPos + 1) {
345 ++LastPos;
346 } else {
347 LastValueTy = ValueTy;
348 LastPos = pos == 0 ? 0 : -1;
349 }
350
351 switch (ValueTy) {
352 case kUnknownValueType:
353 printf(format: "__ ");
354 break;
355 case kFloatValueType:
356 printf(format: "f%x ", pos);
357 if (LastPos == sizeof(float) - 1) {
358 decoded_values[num_decoded_values] =
359 ReadShadow(ptr: shadow + kShadowScale * (Offset + 1 - sizeof(float)),
360 ShadowTypeId: static_cast<char>(shadow_value_type_ids & 0xff));
361 ++num_decoded_values;
362 }
363 break;
364 case kDoubleValueType:
365 printf(format: "d%x ", pos);
366 if (LastPos == sizeof(double) - 1) {
367 decoded_values[num_decoded_values] = ReadShadow(
368 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(double)),
369 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 8) & 0xff));
370 ++num_decoded_values;
371 }
372 break;
373 case kFp80ValueType:
374 printf(format: "l%x ", pos);
375 if (LastPos == sizeof(long double) - 1) {
376 decoded_values[num_decoded_values] = ReadShadow(
377 ptr: shadow + kShadowScale * (Offset + 1 - sizeof(long double)),
378 ShadowTypeId: static_cast<char>((shadow_value_type_ids >> 16) & 0xff));
379 ++num_decoded_values;
380 }
381 break;
382 }
383 ++Offset;
384 }
385 for (int i = 0; i < num_decoded_values; ++i) {
386 printf(format: " (%s)", FTPrinter<__float128>::dec(value: decoded_values[i]).Buffer);
387 }
388 num_decoded_values = 0;
389 printf(format: "\n");
390 }
391}
392
393alignas(16) SANITIZER_INTERFACE_ATTRIBUTE
394 thread_local uptr __nsan_shadow_ret_tag = 0;
395
396alignas(16) SANITIZER_INTERFACE_ATTRIBUTE
397 thread_local char __nsan_shadow_ret_ptr[kMaxVectorWidth *
398 sizeof(__float128)];
399
400alignas(16) SANITIZER_INTERFACE_ATTRIBUTE
401 thread_local uptr __nsan_shadow_args_tag = 0;
402
403// Maximum number of args. This should be enough for anyone (tm). An alternate
404// scheme is to have the generated code create an alloca and make
405// __nsan_shadow_args_ptr point ot the alloca.
406constexpr const int kMaxNumArgs = 128;
407alignas(16) SANITIZER_INTERFACE_ATTRIBUTE
408 thread_local char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs *
409 sizeof(__float128)];
410
411enum ContinuationType { // Keep in sync with instrumentation pass.
412 kContinueWithShadow = 0,
413 kResumeFromValue = 1,
414};
415
416// Checks the consistency between application and shadow value. Returns true
417// when the instrumented code should resume computations from the original value
418// rather than the shadow value. This prevents one error to propagate to all
419// subsequent operations. This behaviour is tunable with flags.
420template <typename FT, typename ShadowFT>
421int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,
422 uptr CheckArg) {
423 // We do all comparisons in the InternalFT domain, which is the largest FT
424 // type.
425 using InternalFT = LargestFT<FT, ShadowFT>;
426 const InternalFT check_value = value;
427 const InternalFT check_shadow = Shadow;
428
429 // See this article for an interesting discussion of how to compare floats:
430 // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
431 static constexpr const FT Eps = FTInfo<FT>::kEpsilon;
432
433 const InternalFT abs_err = ftAbs(check_value - check_shadow);
434
435 if (flags().enable_check_stats) {
436 GET_CALLER_PC_BP;
437 // We are re-computing `largest` here because this is a cold branch, and we
438 // want to avoid having to move the computation of `largest` before the
439 // absolute value check when this branch is not taken.
440 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
441 nsan_stats->AddCheck(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
442 }
443
444 // Note: writing the comparison that way ensures that when `abs_err` is Nan
445 // (value and shadow are inf or -inf), we pass the test.
446 if (!(abs_err >= flags().cached_absolute_error_threshold))
447 return kContinueWithShadow;
448
449 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
450 if (abs_err * (1ull << flags().log2_max_relative_error) <= largest)
451 return kContinueWithShadow; // No problem here.
452
453 if (!flags().disable_warnings) {
454 GET_CALLER_PC_BP;
455 BufferedStackTrace stack;
456 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
457 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Consistency)) {
458 // FIXME: optionally print.
459 return flags().resume_after_suppression ? kResumeFromValue
460 : kContinueWithShadow;
461 }
462
463 Decorator D;
464 Printf(format: "%s", D.Warning());
465 // Printf does not support float formatting.
466 char RelErrBuf[64] = "inf";
467 if (largest > Eps) {
468 snprintf(s: RelErrBuf, maxlen: sizeof(RelErrBuf) - 1, format: "%.20Lf%% (2^%.0Lf epsilons)",
469 static_cast<long double>(100.0 * abs_err / largest),
470 log2l(x: static_cast<long double>(abs_err / largest / Eps)));
471 }
472 char ulp_err_buf[128] = "";
473 const double shadow_ulp_diff = GetULPDiff(check_value, check_shadow);
474 if (shadow_ulp_diff != kMaxULPDiff) {
475 // This is the ULP diff in the internal domain. The user actually cares
476 // about that in the original domain.
477 const double ulp_diff =
478 shadow_ulp_diff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -
479 FTInfo<FT>::kMantissaBits));
480 snprintf(s: ulp_err_buf, maxlen: sizeof(ulp_err_buf) - 1,
481 format: "(%.0f ULPs == %.1f digits == %.1f bits)", ulp_diff,
482 log10(x: ulp_diff), log2(x: ulp_diff));
483 }
484 Printf(format: "WARNING: NumericalStabilitySanitizer: inconsistent shadow results");
485 switch (CheckType) {
486 case CheckTypeT::kUnknown:
487 case CheckTypeT::kFcmp:
488 case CheckTypeT::kMaxCheckType:
489 break;
490 case CheckTypeT::kRet:
491 Printf(format: " while checking return value");
492 break;
493 case CheckTypeT::kArg:
494 Printf(format: " while checking call argument #%d", static_cast<int>(CheckArg));
495 break;
496 case CheckTypeT::kLoad:
497 Printf(
498 format: " while checking load from address 0x%lx. This is due to incorrect "
499 "shadow memory tracking, typically due to uninstrumented code "
500 "writing to memory.",
501 CheckArg);
502 break;
503 case CheckTypeT::kStore:
504 Printf(format: " while checking store to address 0x%lx", CheckArg);
505 break;
506 case CheckTypeT::kInsert:
507 Printf(format: " while checking vector insert");
508 break;
509 case CheckTypeT::kUser:
510 Printf(format: " in user-initiated check");
511 break;
512 }
513 using ValuePrinter = FTPrinter<FT>;
514 using ShadowPrinter = FTPrinter<ShadowFT>;
515 Printf(format: "%s", D.Default());
516
517 Printf("\n"
518 "%-12s precision (native): dec: %s hex: %s\n"
519 "%-12s precision (shadow): dec: %s hex: %s\n"
520 "shadow truncated to %-12s: dec: %s hex: %s\n"
521 "Relative error: %s\n"
522 "Absolute error: %s\n"
523 "%s\n",
524 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(value).Buffer,
525 ValuePrinter::hex(value).Buffer, FTInfo<ShadowFT>::kCppTypeName,
526 ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,
527 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,
528 ValuePrinter::hex(Shadow).Buffer, RelErrBuf,
529 ValuePrinter::hex(abs_err).Buffer, ulp_err_buf);
530 stack.Print();
531 }
532
533 if (flags().enable_warning_stats) {
534 GET_CALLER_PC_BP;
535 nsan_stats->AddWarning(check_ty: CheckType, pc, bp, rel_err: abs_err / largest);
536 }
537
538 if (flags().halt_on_error) {
539 if (common_flags()->abort_on_error)
540 Printf(format: "ABORTING\n");
541 else
542 Printf(format: "Exiting\n");
543 Die();
544 }
545 return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
546}
547
548extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(
549 float value, double shadow, int32_t check_type, uptr check_arg) {
550 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
551}
552
553extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(
554 double value, long double shadow, int32_t check_type, uptr check_arg) {
555 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
556}
557
558extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(
559 double value, __float128 shadow, int32_t check_type, uptr check_arg) {
560 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
561}
562
563extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t
564__nsan_internal_check_longdouble_q(long double value, __float128 shadow,
565 int32_t check_type, uptr check_arg) {
566 return checkFT(value, Shadow: shadow, CheckType: static_cast<CheckTypeT>(check_type), CheckArg: check_arg);
567}
568
569static const char *GetTruthValueName(bool v) { return v ? "true" : "false"; }
570
571// This uses the same values as CmpInst::Predicate.
572static const char *GetPredicateName(int v) {
573 switch (v) {
574 case 0:
575 return "(false)";
576 case 1:
577 return "==";
578 case 2:
579 return ">";
580 case 3:
581 return ">=";
582 case 4:
583 return "<";
584 case 5:
585 return "<=";
586 case 6:
587 return "!=";
588 case 7:
589 return "(ordered)";
590 case 8:
591 return "(unordered)";
592 case 9:
593 return "==";
594 case 10:
595 return ">";
596 case 11:
597 return ">=";
598 case 12:
599 return "<";
600 case 13:
601 return "<=";
602 case 14:
603 return "!=";
604 case 15:
605 return "(true)";
606 }
607 return "??";
608}
609
610template <typename FT, typename ShadowFT>
611void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,
612 ShadowFT RhsShadow, int Predicate, bool result,
613 bool ShadowResult) {
614 if (result == ShadowResult) {
615 // When a vector comparison fails, we fail each element of the comparison
616 // to simplify instrumented code. Skip elements where the shadow comparison
617 // gave the same result as the original one.
618 return;
619 }
620
621 GET_CALLER_PC_BP;
622 BufferedStackTrace stack;
623 stack.Unwind(pc, bp, context: nullptr, request_fast: false);
624
625 if (GetSuppressionForStack(Stack: &stack, K: CheckKind::Fcmp)) {
626 // FIXME: optionally print.
627 return;
628 }
629
630 if (flags().enable_warning_stats)
631 nsan_stats->AddWarning(check_ty: CheckTypeT::kFcmp, pc, bp, rel_err: 0.0);
632
633 if (flags().disable_warnings)
634 return;
635
636 // FIXME: ideally we would print the shadow value as FP128. Right now because
637 // we truncate to long double we can sometimes see stuff like:
638 // shadow <value> == <value> (false)
639 using ValuePrinter = FTPrinter<FT>;
640 using ShadowPrinter = FTPrinter<ShadowFT>;
641 Decorator D;
642 const char *const PredicateName = GetPredicateName(v: Predicate);
643 Printf(format: "%s", D.Warning());
644 Printf(format: "WARNING: NumericalStabilitySanitizer: floating-point comparison "
645 "results depend on precision\n");
646 Printf(format: "%s", D.Default());
647 Printf("%-12s precision dec (native): %s %s %s (%s)\n"
648 "%-12s precision dec (shadow): %s %s %s (%s)\n"
649 "%-12s precision hex (native): %s %s %s (%s)\n"
650 "%-12s precision hex (shadow): %s %s %s (%s)\n"
651 "%s",
652 // Native, decimal.
653 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,
654 ValuePrinter::dec(Rhs).Buffer, GetTruthValueName(v: result),
655 // Shadow, decimal
656 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,
657 PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,
658 GetTruthValueName(v: ShadowResult),
659 // Native, hex.
660 FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,
661 ValuePrinter::hex(Rhs).Buffer, GetTruthValueName(v: result),
662 // Shadow, hex
663 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,
664 PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,
665 GetTruthValueName(v: ShadowResult), D.End());
666 stack.Print();
667 if (flags().halt_on_error) {
668 Printf(format: "Exiting\n");
669 Die();
670 }
671}
672
673extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
674__nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,
675 double rhs_shadow, int predicate, bool result,
676 bool shadow_result) {
677 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
678 ShadowResult: shadow_result);
679}
680
681extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
682__nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,
683 __float128 rhs_shadow, int predicate, bool result,
684 bool shadow_result) {
685 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
686 ShadowResult: shadow_result);
687}
688
689extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
690__nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,
691 long double rhs_shadow, int predicate, bool result,
692 bool shadow_result) {
693 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
694 ShadowResult: shadow_result);
695}
696
697extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
698__nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,
699 __float128 lhs_shadow, __float128 rhs_shadow,
700 int predicate, bool result, bool shadow_result) {
701 fCmpFailFT(Lhs: lhs, Rhs: rhs, LhsShadow: lhs_shadow, RhsShadow: rhs_shadow, Predicate: predicate, result,
702 ShadowResult: shadow_result);
703}
704
705template <typename FT> void checkFTFromShadowStack(const FT value) {
706 // Get the shadow 2FT value from the shadow stack. Note that
707 // __nsan_check_{float,double,long double} is a function like any other, so
708 // the instrumentation will have placed the shadow value on the shadow stack.
709 using ShadowFT = typename FTInfo<FT>::shadow_type;
710 ShadowFT Shadow;
711 __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
712 checkFT(value, Shadow, CheckTypeT::kUser, 0);
713}
714
715// FIXME: Add suffixes and let the instrumentation pass automatically add
716// suffixes.
717extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float value) {
718 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&
719 "__nsan_check_float called from non-instrumented function");
720 checkFTFromShadowStack(value);
721}
722
723extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
724__nsan_check_double(double value) {
725 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&
726 "__nsan_check_double called from non-instrumented function");
727 checkFTFromShadowStack(value);
728}
729
730extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
731__nsan_check_longdouble(long double value) {
732 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&
733 "__nsan_check_longdouble called from non-instrumented function");
734 checkFTFromShadowStack(value);
735}
736
737template <typename FT> static void dumpFTFromShadowStack(const FT value) {
738 // Get the shadow 2FT value from the shadow stack. Note that
739 // __nsan_dump_{float,double,long double} is a function like any other, so
740 // the instrumentation will have placed the shadow value on the shadow stack.
741 using ShadowFT = typename FTInfo<FT>::shadow_type;
742 ShadowFT shadow;
743 __builtin_memcpy(&shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
744 using ValuePrinter = FTPrinter<FT>;
745 using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;
746 printf("value dec:%s hex:%s\n"
747 "shadow dec:%s hex:%s\n",
748 ValuePrinter::dec(value).Buffer, ValuePrinter::hex(value).Buffer,
749 ShadowPrinter::dec(shadow).Buffer, ShadowPrinter::hex(shadow).Buffer);
750}
751
752extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float value) {
753 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&
754 "__nsan_dump_float called from non-instrumented function");
755 dumpFTFromShadowStack(value);
756}
757
758extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double value) {
759 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&
760 "__nsan_dump_double called from non-instrumented function");
761 dumpFTFromShadowStack(value);
762}
763
764extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
765__nsan_dump_longdouble(long double value) {
766 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&
767 "__nsan_dump_longdouble called from non-instrumented function");
768 dumpFTFromShadowStack(value);
769}
770
771extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {
772 printf(format: "ret tag: %lx\n", __nsan_shadow_ret_tag);
773 double v;
774 __builtin_memcpy(&v, __nsan_shadow_ret_ptr, sizeof(double));
775 printf(format: "double value: %f\n", v);
776 // FIXME: float128 value.
777}
778
779extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
780 printf(format: "args tag: %lx\n", __nsan_shadow_args_tag);
781}
782
783bool __nsan::nsan_initialized;
784bool __nsan::nsan_init_is_running;
785
786extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
787 CHECK(!nsan_init_is_running);
788 if (nsan_initialized)
789 return;
790 nsan_init_is_running = true;
791
792 InitializeFlags();
793 InitializeSuppressions();
794 InitializePlatformEarly();
795
796 DisableCoreDumperIfNecessary();
797
798 if (!MmapFixedNoReserve(fixed_addr: TypesAddr(), size: UnusedAddr() - TypesAddr()))
799 Die();
800
801 InitializeInterceptors();
802
803 InitializeStats();
804 if (flags().print_stats_on_exit)
805 Atexit(function: NsanAtexit);
806
807 nsan_init_is_running = false;
808 nsan_initialized = true;
809}
810