1//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the instrumentation pass for the numerical sanitizer.
10// Conceptually the pass injects shadow computations using higher precision
11// types and inserts consistency checks. For details see the paper
12// https://arxiv.org/abs/2102.12782.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
17
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Regex.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Transforms/Utils/BasicBlockUtils.h"
39#include "llvm/Transforms/Utils/Instrumentation.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43#include <cstdint>
44
45using namespace llvm;
46
47#define DEBUG_TYPE "nsan"
48
49STATISTIC(NumInstrumentedFTLoads,
50 "Number of instrumented floating-point loads");
51
52STATISTIC(NumInstrumentedFTCalls,
53 "Number of instrumented floating-point calls");
54STATISTIC(NumInstrumentedFTRets,
55 "Number of instrumented floating-point returns");
56STATISTIC(NumInstrumentedFTStores,
57 "Number of instrumented floating-point stores");
58STATISTIC(NumInstrumentedNonFTStores,
59 "Number of instrumented non floating-point stores");
60STATISTIC(
61 NumInstrumentedNonFTMemcpyStores,
62 "Number of instrumented non floating-point stores with memcpy semantics");
63STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
64
65// Using smaller shadow types types can help improve speed. For example, `dlq`
66// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to
67// `dqq`.
68static cl::opt<std::string> ClShadowMapping(
69 "nsan-shadow-type-mapping", cl::init(Val: "dqq"),
70 cl::desc("One shadow type id for each of `float`, `double`, `long double`. "
71 "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "
72 "ppc_fp128 (extended double) respectively. The default is to "
73 "shadow `float` as `double`, and `double` and `x86_fp80` as "
74 "`fp128`"),
75 cl::Hidden);
76
77static cl::opt<bool>
78 ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(Val: true),
79 cl::desc("Instrument floating-point comparisons"),
80 cl::Hidden);
81
82static cl::opt<std::string> ClCheckFunctionsFilter(
83 "check-functions-filter",
84 cl::desc("Only emit checks for arguments of functions "
85 "whose names match the given regular expression"),
86 cl::value_desc("regex"));
87
88static cl::opt<bool> ClTruncateFCmpEq(
89 "nsan-truncate-fcmp-eq", cl::init(Val: true),
90 cl::desc(
91 "This flag controls the behaviour of fcmp equality comparisons."
92 "For equality comparisons such as `x == 0.0f`, we can perform the "
93 "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "
94 " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "
95 "catch the case when `x_shadow` is accurate enough (and therefore "
96 "close enough to zero) so that `trunc(x_shadow)` is zero even though "
97 "both `x` and `x_shadow` are not"),
98 cl::Hidden);
99
100// When there is external, uninstrumented code writing to memory, the shadow
101// memory can get out of sync with the application memory. Enabling this flag
102// emits consistency checks for loads to catch this situation.
103// When everything is instrumented, this is not strictly necessary because any
104// load should have a corresponding store, but can help debug cases when the
105// framework did a bad job at tracking shadow memory modifications by failing on
106// load rather than store.
107// TODO: provide a way to resume computations from the FT value when the load
108// is inconsistent. This ensures that further computations are not polluted.
109static cl::opt<bool> ClCheckLoads("nsan-check-loads",
110 cl::desc("Check floating-point load"),
111 cl::Hidden);
112
113static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(Val: true),
114 cl::desc("Check floating-point stores"),
115 cl::Hidden);
116
117static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(Val: true),
118 cl::desc("Check floating-point return values"),
119 cl::Hidden);
120
121// LLVM may store constant floats as bitcasted ints.
122// It's not really necessary to shadow such stores,
123// if the shadow value is unknown the framework will re-extend it on load
124// anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is
125// impossible to determine the floating-point type based on the size.
126// However, for debugging purposes it can be useful to model such stores.
127static cl::opt<bool> ClPropagateNonFTConstStoresAsFT(
128 "nsan-propagate-non-ft-const-stores-as-ft",
129 cl::desc(
130 "Propagate non floating-point const stores as floating point values."
131 "For debugging purposes only"),
132 cl::Hidden);
133
134constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
135constexpr StringLiteral kNsanInitName("__nsan_init");
136
137// The following values must be kept in sync with the runtime.
138constexpr int kShadowScale = 2;
139constexpr int kMaxVectorWidth = 8;
140constexpr int kMaxNumArgs = 128;
141constexpr int kMaxShadowTypeSizeBytes = 16; // fp128
142
143namespace {
144
145// Defines the characteristics (type id, type, and floating-point semantics)
146// attached for all possible shadow types.
147class ShadowTypeConfig {
148public:
149 static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
150
151 // The LLVM Type corresponding to the shadow type.
152 virtual Type *getType(LLVMContext &Context) const = 0;
153
154 // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
155 virtual char getNsanTypeId() const = 0;
156
157 virtual ~ShadowTypeConfig() = default;
158};
159
160template <char NsanTypeId>
161class ShadowTypeConfigImpl : public ShadowTypeConfig {
162public:
163 char getNsanTypeId() const override { return NsanTypeId; }
164 static constexpr char kNsanTypeId = NsanTypeId;
165};
166
167// `double` (`d`) shadow type.
168class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
169 Type *getType(LLVMContext &Context) const override {
170 return Type::getDoubleTy(C&: Context);
171 }
172};
173
174// `x86_fp80` (`l`) shadow type: X86 long double.
175class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
176 Type *getType(LLVMContext &Context) const override {
177 return Type::getX86_FP80Ty(C&: Context);
178 }
179};
180
181// `fp128` (`q`) shadow type.
182class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
183 Type *getType(LLVMContext &Context) const override {
184 return Type::getFP128Ty(C&: Context);
185 }
186};
187
188// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
189class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
190 Type *getType(LLVMContext &Context) const override {
191 return Type::getPPC_FP128Ty(C&: Context);
192 }
193};
194
195// Creates a ShadowTypeConfig given its type id.
196std::unique_ptr<ShadowTypeConfig>
197ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
198 switch (TypeId) {
199 case F64ShadowConfig::kNsanTypeId:
200 return std::make_unique<F64ShadowConfig>();
201 case F80ShadowConfig::kNsanTypeId:
202 return std::make_unique<F80ShadowConfig>();
203 case F128ShadowConfig::kNsanTypeId:
204 return std::make_unique<F128ShadowConfig>();
205 case PPC128ShadowConfig::kNsanTypeId:
206 return std::make_unique<PPC128ShadowConfig>();
207 }
208 report_fatal_error(reason: "nsan: invalid shadow type id '" + Twine(TypeId) + "'");
209}
210
211// An enum corresponding to shadow value types. Used as indices in arrays, so
212// not an `enum class`.
213enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
214
215// If `FT` corresponds to a primitive FTValueType, return it.
216static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
217 if (FT->isFloatTy())
218 return kFloat;
219 if (FT->isDoubleTy())
220 return kDouble;
221 if (FT->isX86_FP80Ty())
222 return kLongDouble;
223 return {};
224}
225
226// Returns the LLVM type for an FTValueType.
227static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
228 switch (VT) {
229 case kFloat:
230 return Type::getFloatTy(C&: Context);
231 case kDouble:
232 return Type::getDoubleTy(C&: Context);
233 case kLongDouble:
234 return Type::getX86_FP80Ty(C&: Context);
235 case kNumValueTypes:
236 return nullptr;
237 }
238 llvm_unreachable("Unhandled FTValueType enum");
239}
240
241// Returns the type name for an FTValueType.
242static const char *typeNameFromFTValueType(FTValueType VT) {
243 switch (VT) {
244 case kFloat:
245 return "float";
246 case kDouble:
247 return "double";
248 case kLongDouble:
249 return "longdouble";
250 case kNumValueTypes:
251 return nullptr;
252 }
253 llvm_unreachable("Unhandled FTValueType enum");
254}
255
256// A specific mapping configuration of application type to shadow type for nsan
257// (see -nsan-shadow-mapping flag).
258class MappingConfig {
259public:
260 explicit MappingConfig(LLVMContext &C) : Context(C) {
261 if (ClShadowMapping.size() != 3)
262 report_fatal_error(reason: "Invalid nsan mapping: " + Twine(ClShadowMapping));
263 unsigned ShadowTypeSizeBits[kNumValueTypes];
264 for (int VT = 0; VT < kNumValueTypes; ++VT) {
265 auto Config = ShadowTypeConfig::fromNsanTypeId(TypeId: ClShadowMapping[VT]);
266 if (!Config)
267 report_fatal_error(reason: "Failed to get ShadowTypeConfig for " +
268 Twine(ClShadowMapping[VT]));
269 const unsigned AppTypeSize =
270 typeFromFTValueType(VT: static_cast<FTValueType>(VT), Context)
271 ->getScalarSizeInBits();
272 const unsigned ShadowTypeSize =
273 Config->getType(Context)->getScalarSizeInBits();
274 // Check that the shadow type size is at most kShadowScale times the
275 // application type size, so that shadow memory compoutations are valid.
276 if (ShadowTypeSize > kShadowScale * AppTypeSize)
277 report_fatal_error(reason: "Invalid nsan mapping f" + Twine(AppTypeSize) +
278 "->f" + Twine(ShadowTypeSize) +
279 ": The shadow type size should be at most " +
280 Twine(kShadowScale) +
281 " times the application type size");
282 ShadowTypeSizeBits[VT] = ShadowTypeSize;
283 Configs[VT] = std::move(Config);
284 }
285
286 // Check that the mapping is monotonous. This is required because if one
287 // does an fpextend of `float->long double` in application code, nsan is
288 // going to do an fpextend of `shadow(float) -> shadow(long double)` in
289 // shadow code. This will fail in `qql` mode, since nsan would be
290 // fpextending `f128->long`, which is invalid.
291 // TODO: Relax this.
292 if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
293 ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
294 report_fatal_error(reason: "Invalid nsan mapping: { float->f" +
295 Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
296 Twine(ShadowTypeSizeBits[kDouble]) +
297 "; long double->f" +
298 Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
299 }
300
301 const ShadowTypeConfig &byValueType(FTValueType VT) const {
302 assert(VT < FTValueType::kNumValueTypes && "invalid value type");
303 return *Configs[VT];
304 }
305
306 // Returns the extended shadow type for a given application type.
307 Type *getExtendedFPType(Type *FT) const {
308 if (const auto VT = ftValueTypeFromType(FT))
309 return Configs[*VT]->getType(Context);
310 if (FT->isVectorTy()) {
311 auto *VecTy = cast<VectorType>(Val: FT);
312 // TODO: add support for scalable vector types.
313 if (VecTy->isScalableTy())
314 return nullptr;
315 Type *ExtendedScalar = getExtendedFPType(FT: VecTy->getElementType());
316 return ExtendedScalar
317 ? VectorType::get(ElementType: ExtendedScalar, EC: VecTy->getElementCount())
318 : nullptr;
319 }
320 return nullptr;
321 }
322
323private:
324 LLVMContext &Context;
325 std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
326};
327
328// The memory extents of a type specifies how many elements of a given
329// FTValueType needs to be stored when storing this type.
330struct MemoryExtents {
331 FTValueType ValueType;
332 uint64_t NumElts;
333};
334
335static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
336 if (const auto VT = ftValueTypeFromType(FT))
337 return {.ValueType: *VT, .NumElts: 1};
338 if (auto *VecTy = dyn_cast<VectorType>(Val: FT)) {
339 const auto ScalarExtents = getMemoryExtentsOrDie(FT: VecTy->getElementType());
340 return {.ValueType: ScalarExtents.ValueType,
341 .NumElts: ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
342 }
343 llvm_unreachable("invalid value type");
344}
345
346// The location of a check. Passed as parameters to runtime checking functions.
347class CheckLoc {
348public:
349 // Creates a location that references an application memory location.
350 static CheckLoc makeStore(Value *Address) {
351 CheckLoc Result(kStore);
352 Result.Address = Address;
353 return Result;
354 }
355 static CheckLoc makeLoad(Value *Address) {
356 CheckLoc Result(kLoad);
357 Result.Address = Address;
358 return Result;
359 }
360
361 // Creates a location that references an argument, given by id.
362 static CheckLoc makeArg(int ArgId) {
363 CheckLoc Result(kArg);
364 Result.ArgId = ArgId;
365 return Result;
366 }
367
368 // Creates a location that references the return value of a function.
369 static CheckLoc makeRet() { return CheckLoc(kRet); }
370
371 // Creates a location that references a vector insert.
372 static CheckLoc makeInsert() { return CheckLoc(kInsert); }
373
374 // Returns the CheckType of location this refers to, as an integer-typed LLVM
375 // IR value.
376 Value *getType(LLVMContext &C) const {
377 return ConstantInt::get(Ty: Type::getInt32Ty(C), V: static_cast<int>(CheckTy));
378 }
379
380 // Returns a CheckType-specific value representing details of the location
381 // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
382 // IR value.
383 Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
384 switch (CheckTy) {
385 case kUnknown:
386 llvm_unreachable("unknown type");
387 case kRet:
388 case kInsert:
389 return ConstantInt::get(Ty: IntptrTy, V: 0);
390 case kArg:
391 return ConstantInt::get(Ty: IntptrTy, V: ArgId);
392 case kLoad:
393 case kStore:
394 return Builder.CreatePtrToInt(V: Address, DestTy: IntptrTy);
395 }
396 llvm_unreachable("Unhandled CheckType enum");
397 }
398
399private:
400 // Must be kept in sync with the runtime,
401 // see compiler-rt/lib/nsan/nsan_stats.h
402 enum CheckType {
403 kUnknown = 0,
404 kRet,
405 kArg,
406 kLoad,
407 kStore,
408 kInsert,
409 };
410 explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
411
412 Value *Address = nullptr;
413 const CheckType CheckTy;
414 int ArgId = -1;
415};
416
417// A map of LLVM IR values to shadow LLVM IR values.
418class ValueToShadowMap {
419public:
420 explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}
421
422 ValueToShadowMap(const ValueToShadowMap &) = delete;
423 ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;
424
425 // Sets the shadow value for a value. Asserts that the value does not already
426 // have a value.
427 void setShadow(Value &V, Value &Shadow) {
428 [[maybe_unused]] const bool Inserted = Map.try_emplace(Key: &V, Args: &Shadow).second;
429 LLVM_DEBUG({
430 if (!Inserted) {
431 if (auto *I = dyn_cast<Instruction>(&V))
432 errs() << I->getFunction()->getName() << ": ";
433 errs() << "duplicate shadow (" << &V << "): ";
434 V.dump();
435 }
436 });
437 assert(Inserted && "duplicate shadow");
438 }
439
440 // Returns true if the value already has a shadow (including if the value is a
441 // constant). If true, calling getShadow() is valid.
442 bool hasShadow(Value *V) const { return isa<Constant>(Val: V) || Map.contains(Val: V); }
443
444 // Returns the shadow value for a given value. Asserts that the value has
445 // a shadow value. Lazily creates shadows for constant values.
446 Value *getShadow(Value *V) const {
447 if (Constant *C = dyn_cast<Constant>(Val: V))
448 return getShadowConstant(C);
449 return Map.find(Val: V)->second;
450 }
451
452 bool empty() const { return Map.empty(); }
453
454private:
455 // Extends a constant application value to its shadow counterpart.
456 APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
457 bool LosesInfo = false;
458 CV.convert(ToSemantics: To, RM: APFloatBase::rmTowardZero, losesInfo: &LosesInfo);
459 return CV;
460 }
461
462 // Returns the shadow constant for the given application constant.
463 Constant *getShadowConstant(Constant *C) const {
464 if (UndefValue *U = dyn_cast<UndefValue>(Val: C)) {
465 return UndefValue::get(T: Config.getExtendedFPType(FT: U->getType()));
466 }
467 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
468 // Floating-point constants.
469 Type *Ty = Config.getExtendedFPType(FT: CFP->getType());
470 return ConstantFP::get(
471 Ty, V: extendConstantFP(CV: CFP->getValueAPF(),
472 To: Ty->getScalarType()->getFltSemantics()));
473 }
474 // Vector, array, or aggregate constants.
475 if (C->getType()->isVectorTy()) {
476 SmallVector<Constant *, 8> Elements;
477 for (int I = 0, E = cast<VectorType>(Val: C->getType())
478 ->getElementCount()
479 .getFixedValue();
480 I < E; ++I)
481 Elements.push_back(Elt: getShadowConstant(C: C->getAggregateElement(Elt: I)));
482 return ConstantVector::get(V: Elements);
483 }
484 llvm_unreachable("unimplemented");
485 }
486
487 const MappingConfig &Config;
488 DenseMap<Value *, Value *> Map;
489};
490
491class NsanMemOpFn {
492public:
493 NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,
494 size_t NumArgs);
495 FunctionCallee getFunctionFor(uint64_t MemOpSize) const;
496 FunctionCallee getFallback() const;
497
498private:
499 SmallVector<FunctionCallee> Funcs;
500 size_t NumSizedFuncs;
501};
502
503NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,
504 StringRef Fallback, size_t NumArgs) {
505 LLVMContext &Ctx = M.getContext();
506 AttributeList Attr;
507 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
508 Type *PtrTy = PointerType::getUnqual(C&: Ctx);
509 Type *VoidTy = Type::getVoidTy(C&: Ctx);
510 IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx);
511 FunctionType *SizedFnTy = nullptr;
512
513 NumSizedFuncs = Sized.size();
514
515 // First entry is fallback function
516 if (NumArgs == 3) {
517 Funcs.push_back(
518 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy));
519 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy, PtrTy}, isVarArg: false);
520 } else if (NumArgs == 2) {
521 Funcs.push_back(
522 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: IntptrTy));
523 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false);
524 } else {
525 llvm_unreachable("Unexpected value of sized functions arguments");
526 }
527
528 for (size_t i = 0; i < NumSizedFuncs; ++i)
529 Funcs.push_back(Elt: M.getOrInsertFunction(Name: Sized[i], T: SizedFnTy, AttributeList: Attr));
530}
531
532FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {
533 // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the
534 // following code assumes that the number of functions in `Func` is sufficient
535 assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");
536
537 size_t Idx =
538 MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));
539
540 return Funcs[Idx];
541}
542
543FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }
544
545/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
546/// API function declarations into the module if they don't exist already.
547/// Instantiating ensures the __nsan_init function is in the list of global
548/// constructors for the module.
549class NumericalStabilitySanitizer {
550public:
551 NumericalStabilitySanitizer(Module &M);
552 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
553
554private:
555 bool instrumentMemIntrinsic(MemIntrinsic *MI);
556 void maybeAddSuffixForNsanInterface(CallBase *CI);
557 bool addrPointsToConstantData(Value *Addr);
558 void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
559 ValueToShadowMap &Map);
560 Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
561 const TargetLibraryInfo &TLI,
562 const ValueToShadowMap &Map);
563 PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
564 void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
565 ValueToShadowMap &Map);
566
567 void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
568 const ValueToShadowMap &Map);
569
570 void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
571 const ValueToShadowMap &Map);
572 Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
573 CheckLoc Loc);
574 Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
575 CheckLoc Loc);
576 void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
577
578 // Value creation handlers.
579 Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
580 Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
581 const TargetLibraryInfo &TLI,
582 const ValueToShadowMap &Map, IRBuilder<> &Builder);
583 Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
584 const TargetLibraryInfo &TLI,
585 const ValueToShadowMap &Map,
586 IRBuilder<> &Builder);
587 Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
588 const ValueToShadowMap &Map, IRBuilder<> &Builder);
589 Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,
590 const ValueToShadowMap &Map, IRBuilder<> &Builder);
591
592 // Value propagation handlers.
593 void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
594 const ValueToShadowMap &Map);
595 void propagateNonFTStore(StoreInst &Store, Type *VT,
596 const ValueToShadowMap &Map);
597
598 const DataLayout &DL;
599 LLVMContext &Context;
600 MappingConfig Config;
601 IntegerType *IntptrTy = nullptr;
602
603 // TODO: Use std::array instead?
604 FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};
605 FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};
606 FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};
607 FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};
608
609 NsanMemOpFn NsanCopyFns;
610 NsanMemOpFn NsanSetUnknownFns;
611
612 FunctionCallee NsanGetRawShadowTypePtr;
613 FunctionCallee NsanGetRawShadowPtr;
614 GlobalValue *NsanShadowRetTag = nullptr;
615
616 Type *NsanShadowRetType = nullptr;
617 GlobalValue *NsanShadowRetPtr = nullptr;
618
619 GlobalValue *NsanShadowArgsTag = nullptr;
620
621 Type *NsanShadowArgsType = nullptr;
622 GlobalValue *NsanShadowArgsPtr = nullptr;
623
624 std::optional<Regex> CheckFunctionsFilter;
625};
626} // end anonymous namespace
627
628PreservedAnalyses
629NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
630 getOrCreateSanitizerCtorAndInitFunctions(
631 M, CtorName: kNsanModuleCtorName, InitName: kNsanInitName, /*InitArgTypes=*/{},
632 /*InitArgs=*/{},
633 // This callback is invoked when the functions are created the first
634 // time. Hook them into the global ctors list in that case:
635 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
636
637 NumericalStabilitySanitizer Nsan(M);
638 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
639 for (Function &F : M)
640 Nsan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
641
642 return PreservedAnalyses::none();
643}
644
645static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
646 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&M, Ty, Name] {
647 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
648 nullptr, Name, nullptr,
649 GlobalVariable::InitialExecTLSModel);
650 });
651}
652
653NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M)
654 : DL(M.getDataLayout()), Context(M.getContext()), Config(Context),
655 NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},
656 "__nsan_copy_values", /*NumArgs=*/3),
657 NsanSetUnknownFns(M,
658 {"__nsan_set_value_unknown_4",
659 "__nsan_set_value_unknown_8",
660 "__nsan_set_value_unknown_16"},
661 "__nsan_set_value_unknown", /*NumArgs=*/2) {
662 IntptrTy = DL.getIntPtrType(C&: Context);
663 Type *PtrTy = PointerType::getUnqual(C&: Context);
664 Type *Int32Ty = Type::getInt32Ty(C&: Context);
665 Type *Int1Ty = Type::getInt1Ty(C&: Context);
666 Type *VoidTy = Type::getVoidTy(C&: Context);
667
668 AttributeList Attr;
669 Attr = Attr.addFnAttribute(C&: Context, Kind: Attribute::NoUnwind);
670 // Initialize the runtime values (functions and global variables).
671 for (int I = 0; I < kNumValueTypes; ++I) {
672 const FTValueType VT = static_cast<FTValueType>(I);
673 const char *VTName = typeNameFromFTValueType(VT);
674 Type *VTTy = typeFromFTValueType(VT, Context);
675
676 // Load/store.
677 const std::string GetterPrefix =
678 std::string("__nsan_get_shadow_ptr_for_") + VTName;
679 NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
680 Name: GetterPrefix + "_store", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
681 NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
682 Name: GetterPrefix + "_load", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
683
684 // Check.
685 const auto &ShadowConfig = Config.byValueType(VT);
686 Type *ShadowTy = ShadowConfig.getType(Context);
687 NsanCheckValue[VT] =
688 M.getOrInsertFunction(Name: std::string("__nsan_internal_check_") + VTName +
689 "_" + ShadowConfig.getNsanTypeId(),
690 AttributeList: Attr, RetTy: Int32Ty, Args: VTTy, Args: ShadowTy, Args: Int32Ty, Args: IntptrTy);
691 NsanFCmpFail[VT] = M.getOrInsertFunction(
692 Name: std::string("__nsan_fcmp_fail_") + VTName + "_" +
693 ShadowConfig.getNsanTypeId(),
694 AttributeList: Attr, RetTy: VoidTy, Args: VTTy, Args: VTTy, Args: ShadowTy, Args: ShadowTy, Args: Int32Ty, Args: Int1Ty, Args: Int1Ty);
695 }
696
697 // TODO: Add attributes nofree, nosync, readnone, readonly,
698 NsanGetRawShadowTypePtr = M.getOrInsertFunction(
699 Name: "__nsan_internal_get_raw_shadow_type_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
700 NsanGetRawShadowPtr = M.getOrInsertFunction(
701 Name: "__nsan_internal_get_raw_shadow_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
702
703 NsanShadowRetTag = createThreadLocalGV(Name: "__nsan_shadow_ret_tag", M, Ty: IntptrTy);
704
705 NsanShadowRetType = ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
706 NumElements: kMaxVectorWidth * kMaxShadowTypeSizeBytes);
707 NsanShadowRetPtr =
708 createThreadLocalGV(Name: "__nsan_shadow_ret_ptr", M, Ty: NsanShadowRetType);
709
710 NsanShadowArgsTag =
711 createThreadLocalGV(Name: "__nsan_shadow_args_tag", M, Ty: IntptrTy);
712
713 NsanShadowArgsType =
714 ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
715 NumElements: kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
716
717 NsanShadowArgsPtr =
718 createThreadLocalGV(Name: "__nsan_shadow_args_ptr", M, Ty: NsanShadowArgsType);
719
720 if (!ClCheckFunctionsFilter.empty()) {
721 Regex R = Regex(ClCheckFunctionsFilter);
722 std::string RegexError;
723 assert(R.isValid(RegexError));
724 CheckFunctionsFilter = std::move(R);
725 }
726}
727
728// Returns true if the given LLVM Value points to constant data (typically, a
729// global variable reference).
730bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
731 // If this is a GEP, just analyze its pointer operand.
732 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Addr))
733 Addr = GEP->getPointerOperand();
734
735 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr))
736 return GV->isConstant();
737 return false;
738}
739
740// This instruments the function entry to create shadow arguments.
741// Pseudocode:
742// if (this_fn_ptr == __nsan_shadow_args_tag) {
743// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
744// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
745// ...
746// __nsan_shadow_args_tag = 0;
747// } else {
748// s(arg0) = fext(arg0);
749// s(arg1) = fext(arg1);
750// ...
751// }
752void NumericalStabilitySanitizer::createShadowArguments(
753 Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
754 assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
755
756 // Do not bother if there are no FP args.
757 if (all_of(Range: F.args(), P: [this](const Argument &Arg) {
758 return Config.getExtendedFPType(FT: Arg.getType()) == nullptr;
759 }))
760 return;
761
762 IRBuilder<> Builder(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt());
763 // The function has shadow args if the shadow args tag matches the function
764 // address.
765 Value *HasShadowArgs = Builder.CreateICmpEQ(
766 LHS: Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowArgsTag, /*isVolatile=*/false),
767 RHS: Builder.CreatePtrToInt(V: &F, DestTy: IntptrTy));
768
769 unsigned ShadowArgsOffsetBytes = 0;
770 for (Argument &Arg : F.args()) {
771 Type *VT = Arg.getType();
772 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
773 if (ExtendedVT == nullptr)
774 continue; // Not an FT value.
775 Value *L = Builder.CreateAlignedLoad(
776 Ty: ExtendedVT,
777 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
778 Idx1: ShadowArgsOffsetBytes),
779 Align: Align(1), /*isVolatile=*/false);
780 Value *Shadow = Builder.CreateSelect(C: HasShadowArgs, True: L,
781 False: Builder.CreateFPExt(V: &Arg, DestTy: ExtendedVT));
782 Map.setShadow(V&: Arg, Shadow&: *Shadow);
783 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
784 assert(!SlotSize.isScalable() && "unsupported");
785 ShadowArgsOffsetBytes += SlotSize;
786 }
787 Builder.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: 0), Ptr: NsanShadowArgsTag);
788}
789
790// Returns true if the instrumentation should emit code to check arguments
791// before a function call.
792static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
793 const std::optional<Regex> &CheckFunctionsFilter) {
794
795 Function *Fn = CI.getCalledFunction();
796
797 if (CheckFunctionsFilter) {
798 // Skip checking args of indirect calls.
799 if (Fn == nullptr)
800 return false;
801 if (CheckFunctionsFilter->match(String: Fn->getName()))
802 return true;
803 return false;
804 }
805
806 if (Fn == nullptr)
807 return true; // Always check args of indirect calls.
808
809 // Never check nsan functions, the user called them for a reason.
810 if (Fn->getName().starts_with(Prefix: "__nsan_"))
811 return false;
812
813 const auto ID = Fn->getIntrinsicID();
814 LibFunc LFunc = LibFunc::NotLibFunc;
815 // Always check args of unknown functions.
816 if (ID == Intrinsic::ID() && !TLI.getLibFunc(FDecl: *Fn, F&: LFunc))
817 return true;
818
819 // Do not check args of an `fabs` call that is used for a comparison.
820 // This is typically used for `fabs(a-b) < tolerance`, where what matters is
821 // the result of the comparison, which is already caught be the fcmp checks.
822 if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
823 LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
824 for (const auto &U : CI.users())
825 if (isa<CmpInst>(Val: U))
826 return false;
827
828 return true; // Default is check.
829}
830
831// Populates the shadow call stack (which contains shadow values for every
832// floating-point parameter to the function).
833void NumericalStabilitySanitizer::populateShadowStack(
834 CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
835 // Do not create a shadow stack for inline asm.
836 if (CI.isInlineAsm())
837 return;
838
839 // Do not bother if there are no FP args.
840 if (all_of(Range: CI.operands(), P: [this](const Value *Arg) {
841 return Config.getExtendedFPType(FT: Arg->getType()) == nullptr;
842 }))
843 return;
844
845 IRBuilder<> Builder(&CI);
846 SmallVector<Value *, 8> ArgShadows;
847 const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
848 for (auto [ArgIdx, Arg] : enumerate(First: CI.operands())) {
849 if (Config.getExtendedFPType(FT: Arg->getType()) == nullptr)
850 continue; // Not an FT value.
851 Value *ArgShadow = Map.getShadow(V: Arg);
852 ArgShadows.push_back(Elt: ShouldCheckArgs ? emitCheck(V: Arg, ShadowV: ArgShadow, Builder,
853 Loc: CheckLoc::makeArg(ArgId: ArgIdx))
854 : ArgShadow);
855 }
856
857 // Do not create shadow stacks for intrinsics/known lib funcs.
858 if (Function *Fn = CI.getCalledFunction()) {
859 LibFunc LFunc;
860 if (Fn->isIntrinsic() || TLI.getLibFunc(FDecl: *Fn, F&: LFunc))
861 return;
862 }
863
864 // Set the shadow stack tag.
865 Builder.CreateStore(Val: CI.getCalledOperand(), Ptr: NsanShadowArgsTag);
866 TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(ExactSize: 0);
867
868 unsigned ShadowArgId = 0;
869 for (const Value *Arg : CI.operands()) {
870 Type *VT = Arg->getType();
871 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
872 if (ExtendedVT == nullptr)
873 continue; // Not an FT value.
874 Builder.CreateAlignedStore(
875 Val: ArgShadows[ShadowArgId++],
876 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
877 Idx1: ShadowArgsOffsetBytes),
878 Align: Align(1), /*isVolatile=*/false);
879 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
880 assert(!SlotSize.isScalable() && "unsupported");
881 ShadowArgsOffsetBytes += SlotSize;
882 }
883}
884
885// Internal part of emitCheck(). Returns a value that indicates whether
886// computation should continue with the shadow or resume by re-fextending the
887// value.
888enum class ContinuationType { // Keep in sync with runtime.
889 ContinueWithShadow = 0,
890 ResumeFromValue = 1,
891};
892
893Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
894 IRBuilder<> &Builder,
895 CheckLoc Loc) {
896 // Do not emit checks for constant values, this is redundant.
897 if (isa<Constant>(Val: V))
898 return ConstantInt::get(
899 Ty: Builder.getInt32Ty(),
900 V: static_cast<int>(ContinuationType::ContinueWithShadow));
901
902 Type *Ty = V->getType();
903 if (const auto VT = ftValueTypeFromType(FT: Ty))
904 return Builder.CreateCall(
905 Callee: NsanCheckValue[*VT],
906 Args: {V, ShadowV, Loc.getType(C&: Context), Loc.getValue(IntptrTy, Builder)});
907
908 if (Ty->isVectorTy()) {
909 auto *VecTy = cast<VectorType>(Val: Ty);
910 // We currently skip scalable vector types in MappingConfig,
911 // thus we should not encounter any such types here.
912 assert(!VecTy->isScalableTy() &&
913 "Scalable vector types are not supported yet");
914 Value *CheckResult = nullptr;
915 for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
916 // We resume if any element resumes. Another option would be to create a
917 // vector shuffle with the array of ContinueWithShadow, but that is too
918 // complex.
919 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
920 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
921 Value *ComponentCheckResult =
922 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
923 CheckResult = CheckResult
924 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
925 : ComponentCheckResult;
926 }
927 return CheckResult;
928 }
929 if (Ty->isArrayTy()) {
930 Value *CheckResult = nullptr;
931 for (auto I : seq(Size: Ty->getArrayNumElements())) {
932 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
933 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
934 Value *ComponentCheckResult =
935 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
936 CheckResult = CheckResult
937 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
938 : ComponentCheckResult;
939 }
940 return CheckResult;
941 }
942 if (Ty->isStructTy()) {
943 Value *CheckResult = nullptr;
944 for (auto I : seq(Size: Ty->getStructNumElements())) {
945 if (Config.getExtendedFPType(FT: Ty->getStructElementType(N: I)) == nullptr)
946 continue; // Only check FT values.
947 Value *ExtractV = Builder.CreateExtractValue(Agg: V, Idxs: I);
948 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
949 Value *ComponentCheckResult =
950 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
951 CheckResult = CheckResult
952 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
953 : ComponentCheckResult;
954 }
955 if (!CheckResult)
956 return ConstantInt::get(
957 Ty: Builder.getInt32Ty(),
958 V: static_cast<int>(ContinuationType::ContinueWithShadow));
959 return CheckResult;
960 }
961
962 llvm_unreachable("not implemented");
963}
964
965// Inserts a runtime check of V against its shadow value ShadowV.
966// We check values whenever they escape: on return, call, stores, and
967// insertvalue.
968// Returns the shadow value that should be used to continue the computations,
969// depending on the answer from the runtime.
970// TODO: Should we check on select ? phi ?
971Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
972 IRBuilder<> &Builder,
973 CheckLoc Loc) {
974 // Do not emit checks for constant values, this is redundant.
975 if (isa<Constant>(Val: V))
976 return ShadowV;
977
978 if (Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
979 Function *F = Inst->getFunction();
980 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName())) {
981 return ShadowV;
982 }
983 }
984
985 Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
986 Value *ICmpEQ = Builder.CreateICmpEQ(
987 LHS: CheckResult,
988 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
989 V: static_cast<int>(ContinuationType::ResumeFromValue)));
990 return Builder.CreateSelect(
991 C: ICmpEQ, True: Builder.CreateFPExt(V, DestTy: Config.getExtendedFPType(FT: V->getType())),
992 False: ShadowV);
993}
994
995// Inserts a check that fcmp on shadow values are consistent with that on base
996// values.
997void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
998 const ValueToShadowMap &Map) {
999 if (!ClInstrumentFCmp)
1000 return;
1001
1002 Function *F = FCmp.getFunction();
1003 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName()))
1004 return;
1005
1006 Value *LHS = FCmp.getOperand(i_nocapture: 0);
1007 if (Config.getExtendedFPType(FT: LHS->getType()) == nullptr)
1008 return;
1009 Value *RHS = FCmp.getOperand(i_nocapture: 1);
1010
1011 // Split the basic block. On mismatch, we'll jump to the new basic block with
1012 // a call to the runtime for error reporting.
1013 BasicBlock *FCmpBB = FCmp.getParent();
1014 BasicBlock *NextBB = FCmpBB->splitBasicBlock(I: FCmp.getNextNode());
1015 // Remove the newly created terminator unconditional branch.
1016 FCmpBB->back().eraseFromParent();
1017 BasicBlock *FailBB =
1018 BasicBlock::Create(Context, Name: "", Parent: FCmpBB->getParent(), InsertBefore: NextBB);
1019
1020 // Create the shadow fcmp and comparison between the fcmps.
1021 IRBuilder<> FCmpBuilder(FCmpBB);
1022 FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1023 Value *ShadowLHS = Map.getShadow(V: LHS);
1024 Value *ShadowRHS = Map.getShadow(V: RHS);
1025 // See comment on ClTruncateFCmpEq.
1026 if (FCmp.isEquality() && ClTruncateFCmpEq) {
1027 Type *Ty = ShadowLHS->getType();
1028 ShadowLHS = FCmpBuilder.CreateFPExt(
1029 V: FCmpBuilder.CreateFPTrunc(V: ShadowLHS, DestTy: LHS->getType()), DestTy: Ty);
1030 ShadowRHS = FCmpBuilder.CreateFPExt(
1031 V: FCmpBuilder.CreateFPTrunc(V: ShadowRHS, DestTy: RHS->getType()), DestTy: Ty);
1032 }
1033 Value *ShadowFCmp =
1034 FCmpBuilder.CreateFCmp(P: FCmp.getPredicate(), LHS: ShadowLHS, RHS: ShadowRHS);
1035 Value *OriginalAndShadowFcmpMatch =
1036 FCmpBuilder.CreateICmpEQ(LHS: &FCmp, RHS: ShadowFCmp);
1037
1038 if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
1039 // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
1040 // where an element is true if the corresponding elements in original and
1041 // shadow are the same. We want all elements to be 1.
1042 OriginalAndShadowFcmpMatch =
1043 FCmpBuilder.CreateAndReduce(Src: OriginalAndShadowFcmpMatch);
1044 }
1045
1046 // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common
1047 // case.
1048 FCmpBuilder.CreateCondBr(Cond: OriginalAndShadowFcmpMatch, True: NextBB, False: FailBB,
1049 BranchWeights: MDBuilder(Context).createLikelyBranchWeights());
1050
1051 // Fill in FailBB.
1052 IRBuilder<> FailBuilder(FailBB);
1053 FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1054
1055 const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
1056 &FailBuilder](Value *L, Value *R, Value *ShadowL,
1057 Value *ShadowR, Value *Result,
1058 Value *ShadowResult) {
1059 Type *FT = L->getType();
1060 FunctionCallee *Callee = nullptr;
1061 if (FT->isFloatTy()) {
1062 Callee = &(NsanFCmpFail[kFloat]);
1063 } else if (FT->isDoubleTy()) {
1064 Callee = &(NsanFCmpFail[kDouble]);
1065 } else if (FT->isX86_FP80Ty()) {
1066 // TODO: make NsanFCmpFailLongDouble work.
1067 Callee = &(NsanFCmpFail[kDouble]);
1068 L = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1069 R = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1070 } else {
1071 llvm_unreachable("not implemented");
1072 }
1073 FailBuilder.CreateCall(Callee: *Callee, Args: {L, R, ShadowL, ShadowR,
1074 ConstantInt::get(Ty: FCmpBuilder.getInt32Ty(),
1075 V: FCmp.getPredicate()),
1076 Result, ShadowResult});
1077 };
1078 if (LHS->getType()->isVectorTy()) {
1079 for (int I = 0, E = cast<VectorType>(Val: LHS->getType())
1080 ->getElementCount()
1081 .getFixedValue();
1082 I < E; ++I) {
1083 Value *ExtractLHS = FailBuilder.CreateExtractElement(Vec: LHS, Idx: I);
1084 Value *ExtractRHS = FailBuilder.CreateExtractElement(Vec: RHS, Idx: I);
1085 Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(Vec: ShadowLHS, Idx: I);
1086 Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(Vec: ShadowRHS, Idx: I);
1087 Value *ExtractFCmp = FailBuilder.CreateExtractElement(Vec: &FCmp, Idx: I);
1088 Value *ExtractShadowFCmp =
1089 FailBuilder.CreateExtractElement(Vec: ShadowFCmp, Idx: I);
1090 EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,
1091 ExtractFCmp, ExtractShadowFCmp);
1092 }
1093 } else {
1094 EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
1095 }
1096 FailBuilder.CreateBr(Dest: NextBB);
1097
1098 ++NumInstrumentedFCmp;
1099}
1100
1101// Creates a shadow phi value for any phi that defines a value of FT type.
1102PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
1103 PHINode &Phi, const TargetLibraryInfo &TLI) {
1104 Type *VT = Phi.getType();
1105 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1106 if (ExtendedVT == nullptr)
1107 return nullptr; // Not an FT value.
1108 // The phi operands are shadow values and are not available when the phi is
1109 // created. They will be populated in a final phase, once all shadow values
1110 // have been created.
1111 PHINode *Shadow = PHINode::Create(Ty: ExtendedVT, NumReservedValues: Phi.getNumIncomingValues());
1112 Shadow->insertAfter(InsertPos: Phi.getIterator());
1113 return Shadow;
1114}
1115
1116Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
1117 Type *ExtendedVT) {
1118 IRBuilder<> Builder(Load.getNextNode());
1119 Builder.SetCurrentDebugLocation(Load.getDebugLoc());
1120 if (addrPointsToConstantData(Addr: Load.getPointerOperand())) {
1121 // No need to look into the shadow memory, the value is a constant. Just
1122 // convert from FT to 2FT.
1123 return Builder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1124 }
1125
1126 // if (%shadowptr == &)
1127 // %shadow = fpext %v
1128 // else
1129 // %shadow = load (ptrcast %shadow_ptr))
1130 // Considered options here:
1131 // - Have `NsanGetShadowPtrForLoad` return a fixed address
1132 // &__nsan_unknown_value_shadow_address that is valid to load from, and
1133 // use a select. This has the advantage that the generated IR is simpler.
1134 // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does
1135 // not short-circuit, dereferencing the returned pointer is no longer an
1136 // option, have to split and create a separate basic block. This has the
1137 // advantage of being easier to debug because it crashes if we ever mess
1138 // up.
1139
1140 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1141 Value *ShadowPtr = Builder.CreateCall(
1142 Callee: NsanGetShadowPtrForLoad[Extents.ValueType],
1143 Args: {Load.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1144 ++NumInstrumentedFTLoads;
1145
1146 // Split the basic block.
1147 BasicBlock *LoadBB = Load.getParent();
1148 BasicBlock *NextBB = LoadBB->splitBasicBlock(I: Builder.GetInsertPoint());
1149 // Create the two options for creating the shadow value.
1150 BasicBlock *ShadowLoadBB =
1151 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1152 BasicBlock *FExtBB =
1153 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1154
1155 // Replace the newly created terminator unconditional branch by a conditional
1156 // branch to one of the options.
1157 {
1158 LoadBB->back().eraseFromParent();
1159 IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
1160 LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1161 LoadBBBuilder.CreateCondBr(Cond: LoadBBBuilder.CreateIsNull(Arg: ShadowPtr), True: FExtBB,
1162 False: ShadowLoadBB);
1163 }
1164
1165 // Fill in ShadowLoadBB.
1166 IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
1167 ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1168 Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
1169 Ty: ExtendedVT, Ptr: ShadowPtr, Align: Align(1), isVolatile: Load.isVolatile());
1170 if (ClCheckLoads) {
1171 ShadowLoad = emitCheck(V: &Load, ShadowV: ShadowLoad, Builder&: ShadowLoadBBBuilder,
1172 Loc: CheckLoc::makeLoad(Address: Load.getPointerOperand()));
1173 }
1174 ShadowLoadBBBuilder.CreateBr(Dest: NextBB);
1175
1176 // Fill in FExtBB.
1177 IRBuilder<> FExtBBBuilder(FExtBB);
1178 FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1179 Value *FExt = FExtBBBuilder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1180 FExtBBBuilder.CreateBr(Dest: NextBB);
1181
1182 // The shadow value come from any of the options.
1183 IRBuilder<> NextBBBuilder(&*NextBB->begin());
1184 NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1185 PHINode *ShadowPhi = NextBBBuilder.CreatePHI(Ty: ExtendedVT, NumReservedValues: 2);
1186 ShadowPhi->addIncoming(V: ShadowLoad, BB: ShadowLoadBB);
1187 ShadowPhi->addIncoming(V: FExt, BB: FExtBB);
1188 return ShadowPhi;
1189}
1190
1191Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,
1192 Type *VT, Type *ExtendedVT,
1193 const ValueToShadowMap &Map,
1194 IRBuilder<> &Builder) {
1195 Value *OrigSource = Trunc.getOperand(i_nocapture: 0);
1196 Type *OrigSourceTy = OrigSource->getType();
1197 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1198
1199 // When truncating:
1200 // - (A) If the source has a shadow, we truncate from the shadow, else we
1201 // truncate from the original source.
1202 // - (B) If the shadow of the source is larger than the shadow of the dest,
1203 // we still need a truncate. Else, the shadow of the source is the same
1204 // type as the shadow of the dest (because mappings are non-decreasing), so
1205 // we don't need to emit a truncate.
1206 // Examples,
1207 // with a mapping of {f32->f64;f64->f80;f80->f128}
1208 // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double
1209 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1210 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1211 // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1)
1212 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80
1213 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1214 // with a mapping of {f32->f64;f64->f128;f80->f128}
1215 // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double
1216 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1217 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1218 // fptrunc x86_fp80 %1 to double -> fp128 %1
1219 // fptrunc fp128 %1 to double -> fp128 %1
1220 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1221 // with a mapping of {f32->f32;f64->f32;f80->f64}
1222 // fptrunc double %1 to float -> float s(%1)
1223 // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float
1224 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float
1225 // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float
1226 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float
1227 // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double
1228
1229 // See (A) above.
1230 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1231 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1232 // See (B) above.
1233 if (SourceTy == ExtendedVT)
1234 return Source;
1235
1236 return Builder.CreateFPTrunc(V: Source, DestTy: ExtendedVT);
1237}
1238
1239Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,
1240 Type *ExtendedVT,
1241 const ValueToShadowMap &Map,
1242 IRBuilder<> &Builder) {
1243 Value *OrigSource = Ext.getOperand(i_nocapture: 0);
1244 Type *OrigSourceTy = OrigSource->getType();
1245 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1246 // When extending:
1247 // - (A) If the source has a shadow, we extend from the shadow, else we
1248 // extend from the original source.
1249 // - (B) If the shadow of the dest is larger than the shadow of the source,
1250 // we still need an extend. Else, the shadow of the source is the same
1251 // type as the shadow of the dest (because mappings are non-decreasing), so
1252 // we don't need to emit an extend.
1253 // Examples,
1254 // with a mapping of {f32->f64;f64->f80;f80->f128}
1255 // fpext half %1 to float -> fpext half %1 to double
1256 // fpext half %1 to double -> fpext half %1 to x86_fp80
1257 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1258 // fpext float %1 to double -> double s(%1)
1259 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1260 // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128
1261 // with a mapping of {f32->f64;f64->f128;f80->f128}
1262 // fpext half %1 to float -> fpext half %1 to double
1263 // fpext half %1 to double -> fpext half %1 to fp128
1264 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1265 // fpext float %1 to double -> fpext double s(%1) to fp128
1266 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1267 // fpext double %1 to x86_fp80 -> fp128 s(%1)
1268 // with a mapping of {f32->f32;f64->f32;f80->f64}
1269 // fpext half %1 to float -> fpext half %1 to float
1270 // fpext half %1 to double -> fpext half %1 to float
1271 // fpext half %1 to x86_fp80 -> fpext half %1 to double
1272 // fpext float %1 to double -> s(%1)
1273 // fpext float %1 to x86_fp80 -> fpext float s(%1) to double
1274 // fpext double %1 to x86_fp80 -> fpext float s(%1) to double
1275
1276 // See (A) above.
1277 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1278 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1279 // See (B) above.
1280 if (SourceTy == ExtendedVT)
1281 return Source;
1282
1283 return Builder.CreateFPExt(V: Source, DestTy: ExtendedVT);
1284}
1285
1286namespace {
1287// TODO: This should be tablegen-ed.
1288struct KnownIntrinsic {
1289 struct WidenedIntrinsic {
1290 const char *NarrowName;
1291 Intrinsic::ID ID; // wide id.
1292 using FnTypeFactory = FunctionType *(*)(LLVMContext &);
1293 FnTypeFactory MakeFnTy;
1294 };
1295
1296 static const char *get(LibFunc LFunc);
1297
1298 // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
1299 // that applies the same operation on the shadow argument.
1300 // Options are:
1301 // - pass in the ID and full function type,
1302 // - pass in the name, which includes the function type through mangling.
1303 static const WidenedIntrinsic *widen(StringRef Name);
1304
1305private:
1306 struct LFEntry {
1307 LibFunc LFunc;
1308 const char *IntrinsicName;
1309 };
1310 static const LFEntry kLibfuncIntrinsics[];
1311
1312 static const WidenedIntrinsic kWidenedIntrinsics[];
1313};
1314} // namespace
1315
1316static FunctionType *makeDoubleDouble(LLVMContext &C) {
1317 return FunctionType::get(Result: Type::getDoubleTy(C), Params: {Type::getDoubleTy(C)}, isVarArg: false);
1318}
1319
1320static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {
1321 return FunctionType::get(Result: Type::getX86_FP80Ty(C), Params: {Type::getX86_FP80Ty(C)},
1322 isVarArg: false);
1323}
1324
1325static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {
1326 return FunctionType::get(Result: Type::getDoubleTy(C),
1327 Params: {Type::getDoubleTy(C), Type::getInt32Ty(C)}, isVarArg: false);
1328}
1329
1330static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {
1331 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1332 Params: {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
1333 isVarArg: false);
1334}
1335
1336static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {
1337 return FunctionType::get(Result: Type::getDoubleTy(C),
1338 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C)}, isVarArg: false);
1339}
1340
1341static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {
1342 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1343 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1344 isVarArg: false);
1345}
1346
1347static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {
1348 return FunctionType::get(
1349 Result: Type::getDoubleTy(C),
1350 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
1351 isVarArg: false);
1352}
1353
1354static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {
1355 return FunctionType::get(
1356 Result: Type::getX86_FP80Ty(C),
1357 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1358 isVarArg: false);
1359}
1360
1361const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
1362 // TODO: Right now we ignore vector intrinsics.
1363 // This is hard because we have to model the semantics of the intrinsics,
1364 // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
1365 // Intrinsics that take any non-vector FT types:
1366 // NOTE: Right now because of
1367 // https://github.com/llvm/llvm-project/issues/44744
1368 // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and
1369 // come back).
1370 {.NarrowName: "llvm.sqrt.f32", .ID: Intrinsic::sqrt, .MakeFnTy: makeDoubleDouble},
1371 {.NarrowName: "llvm.sqrt.f64", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1372 {.NarrowName: "llvm.sqrt.f80", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1373 {.NarrowName: "llvm.powi.f32", .ID: Intrinsic::powi, .MakeFnTy: makeDoubleDoubleI32},
1374 {.NarrowName: "llvm.powi.f64", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1375 {.NarrowName: "llvm.powi.f80", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1376 {.NarrowName: "llvm.sin.f32", .ID: Intrinsic::sin, .MakeFnTy: makeDoubleDouble},
1377 {.NarrowName: "llvm.sin.f64", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1378 {.NarrowName: "llvm.sin.f80", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1379 {.NarrowName: "llvm.cos.f32", .ID: Intrinsic::cos, .MakeFnTy: makeDoubleDouble},
1380 {.NarrowName: "llvm.cos.f64", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1381 {.NarrowName: "llvm.cos.f80", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1382 {.NarrowName: "llvm.pow.f32", .ID: Intrinsic::pow, .MakeFnTy: makeDoubleDoubleDouble},
1383 {.NarrowName: "llvm.pow.f64", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1384 {.NarrowName: "llvm.pow.f80", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1385 {.NarrowName: "llvm.exp.f32", .ID: Intrinsic::exp, .MakeFnTy: makeDoubleDouble},
1386 {.NarrowName: "llvm.exp.f64", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1387 {.NarrowName: "llvm.exp.f80", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1388 {.NarrowName: "llvm.exp2.f32", .ID: Intrinsic::exp2, .MakeFnTy: makeDoubleDouble},
1389 {.NarrowName: "llvm.exp2.f64", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1390 {.NarrowName: "llvm.exp2.f80", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1391 {.NarrowName: "llvm.log.f32", .ID: Intrinsic::log, .MakeFnTy: makeDoubleDouble},
1392 {.NarrowName: "llvm.log.f64", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1393 {.NarrowName: "llvm.log.f80", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1394 {.NarrowName: "llvm.log10.f32", .ID: Intrinsic::log10, .MakeFnTy: makeDoubleDouble},
1395 {.NarrowName: "llvm.log10.f64", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1396 {.NarrowName: "llvm.log10.f80", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1397 {.NarrowName: "llvm.log2.f32", .ID: Intrinsic::log2, .MakeFnTy: makeDoubleDouble},
1398 {.NarrowName: "llvm.log2.f64", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1399 {.NarrowName: "llvm.log2.f80", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1400 {.NarrowName: "llvm.fma.f32", .ID: Intrinsic::fma, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1401
1402 {.NarrowName: "llvm.fmuladd.f32", .ID: Intrinsic::fmuladd, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1403
1404 {.NarrowName: "llvm.fma.f64", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1405
1406 {.NarrowName: "llvm.fmuladd.f64", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1407
1408 {.NarrowName: "llvm.fma.f80", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1409 {.NarrowName: "llvm.fabs.f32", .ID: Intrinsic::fabs, .MakeFnTy: makeDoubleDouble},
1410 {.NarrowName: "llvm.fabs.f64", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1411 {.NarrowName: "llvm.fabs.f80", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1412 {.NarrowName: "llvm.minnum.f32", .ID: Intrinsic::minnum, .MakeFnTy: makeDoubleDoubleDouble},
1413 {.NarrowName: "llvm.minnum.f64", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1414 {.NarrowName: "llvm.minnum.f80", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1415 {.NarrowName: "llvm.maxnum.f32", .ID: Intrinsic::maxnum, .MakeFnTy: makeDoubleDoubleDouble},
1416 {.NarrowName: "llvm.maxnum.f64", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1417 {.NarrowName: "llvm.maxnum.f80", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1418 {.NarrowName: "llvm.minimum.f32", .ID: Intrinsic::minimum, .MakeFnTy: makeDoubleDoubleDouble},
1419 {.NarrowName: "llvm.minimum.f64", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1420 {.NarrowName: "llvm.minimum.f80", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1421 {.NarrowName: "llvm.maximum.f32", .ID: Intrinsic::maximum, .MakeFnTy: makeDoubleDoubleDouble},
1422 {.NarrowName: "llvm.maximum.f64", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1423 {.NarrowName: "llvm.maximum.f80", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1424 {.NarrowName: "llvm.copysign.f32", .ID: Intrinsic::copysign, .MakeFnTy: makeDoubleDoubleDouble},
1425 {.NarrowName: "llvm.copysign.f64", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1426 {.NarrowName: "llvm.copysign.f80", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1427 {.NarrowName: "llvm.floor.f32", .ID: Intrinsic::floor, .MakeFnTy: makeDoubleDouble},
1428 {.NarrowName: "llvm.floor.f64", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1429 {.NarrowName: "llvm.floor.f80", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1430 {.NarrowName: "llvm.ceil.f32", .ID: Intrinsic::ceil, .MakeFnTy: makeDoubleDouble},
1431 {.NarrowName: "llvm.ceil.f64", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1432 {.NarrowName: "llvm.ceil.f80", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1433 {.NarrowName: "llvm.trunc.f32", .ID: Intrinsic::trunc, .MakeFnTy: makeDoubleDouble},
1434 {.NarrowName: "llvm.trunc.f64", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1435 {.NarrowName: "llvm.trunc.f80", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1436 {.NarrowName: "llvm.rint.f32", .ID: Intrinsic::rint, .MakeFnTy: makeDoubleDouble},
1437 {.NarrowName: "llvm.rint.f64", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1438 {.NarrowName: "llvm.rint.f80", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1439 {.NarrowName: "llvm.nearbyint.f32", .ID: Intrinsic::nearbyint, .MakeFnTy: makeDoubleDouble},
1440 {.NarrowName: "llvm.nearbyint.f64", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1441 {.NarrowName: "llvm.nearbyin80f64", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1442 {.NarrowName: "llvm.round.f32", .ID: Intrinsic::round, .MakeFnTy: makeDoubleDouble},
1443 {.NarrowName: "llvm.round.f64", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1444 {.NarrowName: "llvm.round.f80", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1445 {.NarrowName: "llvm.lround.f32", .ID: Intrinsic::lround, .MakeFnTy: makeDoubleDouble},
1446 {.NarrowName: "llvm.lround.f64", .ID: Intrinsic::lround, .MakeFnTy: makeX86FP80X86FP80},
1447 {.NarrowName: "llvm.lround.f80", .ID: Intrinsic::lround, .MakeFnTy: makeX86FP80X86FP80},
1448 {.NarrowName: "llvm.llround.f32", .ID: Intrinsic::llround, .MakeFnTy: makeDoubleDouble},
1449 {.NarrowName: "llvm.llround.f64", .ID: Intrinsic::llround, .MakeFnTy: makeX86FP80X86FP80},
1450 {.NarrowName: "llvm.llround.f80", .ID: Intrinsic::llround, .MakeFnTy: makeX86FP80X86FP80},
1451 {.NarrowName: "llvm.lrint.f32", .ID: Intrinsic::lrint, .MakeFnTy: makeDoubleDouble},
1452 {.NarrowName: "llvm.lrint.f64", .ID: Intrinsic::lrint, .MakeFnTy: makeX86FP80X86FP80},
1453 {.NarrowName: "llvm.lrint.f80", .ID: Intrinsic::lrint, .MakeFnTy: makeX86FP80X86FP80},
1454 {.NarrowName: "llvm.llrint.f32", .ID: Intrinsic::llrint, .MakeFnTy: makeDoubleDouble},
1455 {.NarrowName: "llvm.llrint.f64", .ID: Intrinsic::llrint, .MakeFnTy: makeX86FP80X86FP80},
1456 {.NarrowName: "llvm.llrint.f80", .ID: Intrinsic::llrint, .MakeFnTy: makeX86FP80X86FP80},
1457};
1458
1459const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {
1460 {.LFunc: LibFunc_sqrtf, .IntrinsicName: "llvm.sqrt.f32"},
1461 {.LFunc: LibFunc_sqrt, .IntrinsicName: "llvm.sqrt.f64"},
1462 {.LFunc: LibFunc_sqrtl, .IntrinsicName: "llvm.sqrt.f80"},
1463 {.LFunc: LibFunc_sinf, .IntrinsicName: "llvm.sin.f32"},
1464 {.LFunc: LibFunc_sin, .IntrinsicName: "llvm.sin.f64"},
1465 {.LFunc: LibFunc_sinl, .IntrinsicName: "llvm.sin.f80"},
1466 {.LFunc: LibFunc_cosf, .IntrinsicName: "llvm.cos.f32"},
1467 {.LFunc: LibFunc_cos, .IntrinsicName: "llvm.cos.f64"},
1468 {.LFunc: LibFunc_cosl, .IntrinsicName: "llvm.cos.f80"},
1469 {.LFunc: LibFunc_powf, .IntrinsicName: "llvm.pow.f32"},
1470 {.LFunc: LibFunc_pow, .IntrinsicName: "llvm.pow.f64"},
1471 {.LFunc: LibFunc_powl, .IntrinsicName: "llvm.pow.f80"},
1472 {.LFunc: LibFunc_expf, .IntrinsicName: "llvm.exp.f32"},
1473 {.LFunc: LibFunc_exp, .IntrinsicName: "llvm.exp.f64"},
1474 {.LFunc: LibFunc_expl, .IntrinsicName: "llvm.exp.f80"},
1475 {.LFunc: LibFunc_exp2f, .IntrinsicName: "llvm.exp2.f32"},
1476 {.LFunc: LibFunc_exp2, .IntrinsicName: "llvm.exp2.f64"},
1477 {.LFunc: LibFunc_exp2l, .IntrinsicName: "llvm.exp2.f80"},
1478 {.LFunc: LibFunc_logf, .IntrinsicName: "llvm.log.f32"},
1479 {.LFunc: LibFunc_log, .IntrinsicName: "llvm.log.f64"},
1480 {.LFunc: LibFunc_logl, .IntrinsicName: "llvm.log.f80"},
1481 {.LFunc: LibFunc_log10f, .IntrinsicName: "llvm.log10.f32"},
1482 {.LFunc: LibFunc_log10, .IntrinsicName: "llvm.log10.f64"},
1483 {.LFunc: LibFunc_log10l, .IntrinsicName: "llvm.log10.f80"},
1484 {.LFunc: LibFunc_log2f, .IntrinsicName: "llvm.log2.f32"},
1485 {.LFunc: LibFunc_log2, .IntrinsicName: "llvm.log2.f64"},
1486 {.LFunc: LibFunc_log2l, .IntrinsicName: "llvm.log2.f80"},
1487 {.LFunc: LibFunc_fabsf, .IntrinsicName: "llvm.fabs.f32"},
1488 {.LFunc: LibFunc_fabs, .IntrinsicName: "llvm.fabs.f64"},
1489 {.LFunc: LibFunc_fabsl, .IntrinsicName: "llvm.fabs.f80"},
1490 {.LFunc: LibFunc_copysignf, .IntrinsicName: "llvm.copysign.f32"},
1491 {.LFunc: LibFunc_copysign, .IntrinsicName: "llvm.copysign.f64"},
1492 {.LFunc: LibFunc_copysignl, .IntrinsicName: "llvm.copysign.f80"},
1493 {.LFunc: LibFunc_floorf, .IntrinsicName: "llvm.floor.f32"},
1494 {.LFunc: LibFunc_floor, .IntrinsicName: "llvm.floor.f64"},
1495 {.LFunc: LibFunc_floorl, .IntrinsicName: "llvm.floor.f80"},
1496 {.LFunc: LibFunc_fmaxf, .IntrinsicName: "llvm.maxnum.f32"},
1497 {.LFunc: LibFunc_fmax, .IntrinsicName: "llvm.maxnum.f64"},
1498 {.LFunc: LibFunc_fmaxl, .IntrinsicName: "llvm.maxnum.f80"},
1499 {.LFunc: LibFunc_fminf, .IntrinsicName: "llvm.minnum.f32"},
1500 {.LFunc: LibFunc_fmin, .IntrinsicName: "llvm.minnum.f64"},
1501 {.LFunc: LibFunc_fminl, .IntrinsicName: "llvm.minnum.f80"},
1502 {.LFunc: LibFunc_ceilf, .IntrinsicName: "llvm.ceil.f32"},
1503 {.LFunc: LibFunc_ceil, .IntrinsicName: "llvm.ceil.f64"},
1504 {.LFunc: LibFunc_ceill, .IntrinsicName: "llvm.ceil.f80"},
1505 {.LFunc: LibFunc_truncf, .IntrinsicName: "llvm.trunc.f32"},
1506 {.LFunc: LibFunc_trunc, .IntrinsicName: "llvm.trunc.f64"},
1507 {.LFunc: LibFunc_truncl, .IntrinsicName: "llvm.trunc.f80"},
1508 {.LFunc: LibFunc_rintf, .IntrinsicName: "llvm.rint.f32"},
1509 {.LFunc: LibFunc_rint, .IntrinsicName: "llvm.rint.f64"},
1510 {.LFunc: LibFunc_rintl, .IntrinsicName: "llvm.rint.f80"},
1511 {.LFunc: LibFunc_nearbyintf, .IntrinsicName: "llvm.nearbyint.f32"},
1512 {.LFunc: LibFunc_nearbyint, .IntrinsicName: "llvm.nearbyint.f64"},
1513 {.LFunc: LibFunc_nearbyintl, .IntrinsicName: "llvm.nearbyint.f80"},
1514 {.LFunc: LibFunc_roundf, .IntrinsicName: "llvm.round.f32"},
1515 {.LFunc: LibFunc_round, .IntrinsicName: "llvm.round.f64"},
1516 {.LFunc: LibFunc_roundl, .IntrinsicName: "llvm.round.f80"},
1517};
1518
1519const char *KnownIntrinsic::get(LibFunc LFunc) {
1520 for (const auto &E : kLibfuncIntrinsics) {
1521 if (E.LFunc == LFunc)
1522 return E.IntrinsicName;
1523 }
1524 return nullptr;
1525}
1526
1527const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {
1528 for (const auto &E : kWidenedIntrinsics) {
1529 if (E.NarrowName == Name)
1530 return &E;
1531 }
1532 return nullptr;
1533}
1534
1535// Returns the name of the LLVM intrinsic corresponding to the given function.
1536static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,
1537 const TargetLibraryInfo &TLI) {
1538 LibFunc LFunc;
1539 if (!TLI.getLibFunc(FDecl: Fn, F&: LFunc))
1540 return nullptr;
1541
1542 if (const char *Name = KnownIntrinsic::get(LFunc))
1543 return Name;
1544
1545 LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");
1546 return nullptr;
1547}
1548
1549// Try to handle a known function call.
1550Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(
1551 CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,
1552 const ValueToShadowMap &Map, IRBuilder<> &Builder) {
1553 Function *Fn = Call.getCalledFunction();
1554 if (Fn == nullptr)
1555 return nullptr;
1556
1557 Intrinsic::ID WidenedId = Intrinsic::ID();
1558 FunctionType *WidenedFnTy = nullptr;
1559 if (const auto ID = Fn->getIntrinsicID()) {
1560 const auto *Widened = KnownIntrinsic::widen(Name: Fn->getName());
1561 if (Widened) {
1562 WidenedId = Widened->ID;
1563 WidenedFnTy = Widened->MakeFnTy(Context);
1564 } else {
1565 // If we don't know how to widen the intrinsic, we have no choice but to
1566 // call the non-wide version on a truncated shadow and extend again
1567 // afterwards.
1568 WidenedId = ID;
1569 WidenedFnTy = Fn->getFunctionType();
1570 }
1571 } else if (const char *Name = getIntrinsicFromLibfunc(Fn&: *Fn, VT, TLI)) {
1572 // We might have a call to a library function that we can replace with a
1573 // wider Intrinsic.
1574 const auto *Widened = KnownIntrinsic::widen(Name);
1575 assert(Widened && "make sure KnownIntrinsic entries are consistent");
1576 WidenedId = Widened->ID;
1577 WidenedFnTy = Widened->MakeFnTy(Context);
1578 } else {
1579 // This is not a known library function or intrinsic.
1580 return nullptr;
1581 }
1582
1583 // Check that the widened intrinsic is valid.
1584 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1585 getIntrinsicInfoTableEntries(id: WidenedId, T&: Table);
1586 SmallVector<Type *, 4> ArgTys;
1587 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
1588 [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult MatchResult =
1589 Intrinsic::matchIntrinsicSignature(FTy: WidenedFnTy, Infos&: TableRef, ArgTys);
1590 assert(MatchResult == Intrinsic::MatchIntrinsicTypes_Match &&
1591 "invalid widened intrinsic");
1592 // For known intrinsic functions, we create a second call to the same
1593 // intrinsic with a different type.
1594 SmallVector<Value *, 4> Args;
1595 // The last operand is the intrinsic itself, skip it.
1596 for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {
1597 Value *Arg = Call.getOperand(i_nocapture: I);
1598 Type *OrigArgTy = Arg->getType();
1599 Type *IntrinsicArgTy = WidenedFnTy->getParamType(i: I);
1600 if (OrigArgTy == IntrinsicArgTy) {
1601 Args.push_back(Elt: Arg); // The arg is passed as is.
1602 continue;
1603 }
1604 Type *ShadowArgTy = Config.getExtendedFPType(FT: Arg->getType());
1605 assert(ShadowArgTy &&
1606 "don't know how to get the shadow value for a non-FT");
1607 Value *Shadow = Map.getShadow(V: Arg);
1608 if (ShadowArgTy == IntrinsicArgTy) {
1609 // The shadow is the right type for the intrinsic.
1610 assert(Shadow->getType() == ShadowArgTy);
1611 Args.push_back(Elt: Shadow);
1612 continue;
1613 }
1614 // There is no intrinsic with his level of precision, truncate the shadow.
1615 Args.push_back(Elt: Builder.CreateFPTrunc(V: Shadow, DestTy: IntrinsicArgTy));
1616 }
1617 Value *IntrinsicCall = Builder.CreateIntrinsic(ID: WidenedId, Types: ArgTys, Args);
1618 return WidenedFnTy->getReturnType() == ExtendedVT
1619 ? IntrinsicCall
1620 : Builder.CreateFPExt(V: IntrinsicCall, DestTy: ExtendedVT);
1621}
1622
1623// Handle a CallBase, i.e. a function call, an inline asm sequence, or an
1624// invoke.
1625Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,
1626 Type *ExtendedVT,
1627 const TargetLibraryInfo &TLI,
1628 const ValueToShadowMap &Map,
1629 IRBuilder<> &Builder) {
1630 // We cannot look inside inline asm, just expand the result again.
1631 if (Call.isInlineAsm())
1632 return Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT);
1633
1634 // Intrinsics and library functions (e.g. sin, exp) are handled
1635 // specifically, because we know their semantics and can do better than
1636 // blindly calling them (e.g. compute the sinus in the actual shadow domain).
1637 if (Value *V =
1638 maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))
1639 return V;
1640
1641 // If the return tag matches that of the called function, read the extended
1642 // return value from the shadow ret ptr. Else, just extend the return value.
1643 Value *L =
1644 Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowRetTag, /*isVolatile=*/false);
1645 Value *HasShadowRet = Builder.CreateICmpEQ(
1646 LHS: L, RHS: Builder.CreatePtrToInt(V: Call.getCalledOperand(), DestTy: IntptrTy));
1647
1648 Value *ShadowRetVal = Builder.CreateLoad(
1649 Ty: ExtendedVT,
1650 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0),
1651 /*isVolatile=*/false);
1652 Value *Shadow = Builder.CreateSelect(C: HasShadowRet, True: ShadowRetVal,
1653 False: Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT));
1654 ++NumInstrumentedFTCalls;
1655 return Shadow;
1656}
1657
1658// Creates a shadow value for the given FT value. At that point all operands are
1659// guaranteed to be available.
1660Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(
1661 Instruction &Inst, const TargetLibraryInfo &TLI,
1662 const ValueToShadowMap &Map) {
1663 Type *VT = Inst.getType();
1664 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1665 assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");
1666
1667 if (auto *Load = dyn_cast<LoadInst>(Val: &Inst))
1668 return handleLoad(Load&: *Load, VT, ExtendedVT);
1669
1670 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
1671 // Insert after the call.
1672 BasicBlock::iterator It(Inst);
1673 IRBuilder<> Builder(Call->getParent(), ++It);
1674 Builder.SetCurrentDebugLocation(Call->getDebugLoc());
1675 return handleCallBase(Call&: *Call, VT, ExtendedVT, TLI, Map, Builder);
1676 }
1677
1678 if (auto *Invoke = dyn_cast<InvokeInst>(Val: &Inst)) {
1679 // The Invoke terminates the basic block, create a new basic block in
1680 // between the successful invoke and the next block.
1681 BasicBlock *InvokeBB = Invoke->getParent();
1682 BasicBlock *NextBB = Invoke->getNormalDest();
1683 BasicBlock *NewBB =
1684 BasicBlock::Create(Context, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
1685 Inst.replaceSuccessorWith(OldBB: NextBB, NewBB);
1686
1687 IRBuilder<> Builder(NewBB);
1688 Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());
1689 Value *Shadow = handleCallBase(Call&: *Invoke, VT, ExtendedVT, TLI, Map, Builder);
1690 Builder.CreateBr(Dest: NextBB);
1691 NewBB->replaceSuccessorsPhiUsesWith(Old: InvokeBB, New: NewBB);
1692 return Shadow;
1693 }
1694
1695 IRBuilder<> Builder(Inst.getNextNode());
1696 Builder.SetCurrentDebugLocation(Inst.getDebugLoc());
1697
1698 if (auto *Trunc = dyn_cast<FPTruncInst>(Val: &Inst))
1699 return handleTrunc(Trunc: *Trunc, VT, ExtendedVT, Map, Builder);
1700 if (auto *Ext = dyn_cast<FPExtInst>(Val: &Inst))
1701 return handleExt(Ext: *Ext, VT, ExtendedVT, Map, Builder);
1702
1703 if (auto *UnaryOp = dyn_cast<UnaryOperator>(Val: &Inst))
1704 return Builder.CreateUnOp(Opc: UnaryOp->getOpcode(),
1705 V: Map.getShadow(V: UnaryOp->getOperand(i_nocapture: 0)));
1706
1707 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: &Inst))
1708 return Builder.CreateBinOp(Opc: BinOp->getOpcode(),
1709 LHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 0)),
1710 RHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 1)));
1711
1712 if (isa<UIToFPInst>(Val: &Inst) || isa<SIToFPInst>(Val: &Inst)) {
1713 auto *Cast = cast<CastInst>(Val: &Inst);
1714 return Builder.CreateCast(Op: Cast->getOpcode(), V: Cast->getOperand(i_nocapture: 0),
1715 DestTy: ExtendedVT);
1716 }
1717
1718 if (auto *S = dyn_cast<SelectInst>(Val: &Inst))
1719 return Builder.CreateSelect(C: S->getCondition(),
1720 True: Map.getShadow(V: S->getTrueValue()),
1721 False: Map.getShadow(V: S->getFalseValue()));
1722
1723 if (auto *Freeze = dyn_cast<FreezeInst>(Val: &Inst))
1724 return Builder.CreateFreeze(V: Map.getShadow(V: Freeze->getOperand(i_nocapture: 0)));
1725
1726 if (auto *Extract = dyn_cast<ExtractElementInst>(Val: &Inst))
1727 return Builder.CreateExtractElement(
1728 Vec: Map.getShadow(V: Extract->getVectorOperand()), Idx: Extract->getIndexOperand());
1729
1730 if (auto *Insert = dyn_cast<InsertElementInst>(Val: &Inst))
1731 return Builder.CreateInsertElement(Vec: Map.getShadow(V: Insert->getOperand(i_nocapture: 0)),
1732 NewElt: Map.getShadow(V: Insert->getOperand(i_nocapture: 1)),
1733 Idx: Insert->getOperand(i_nocapture: 2));
1734
1735 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: &Inst))
1736 return Builder.CreateShuffleVector(V1: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 0)),
1737 V2: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 1)),
1738 Mask: Shuffle->getShuffleMask());
1739 // TODO: We could make aggregate object first class citizens. For now we
1740 // just extend the extracted value.
1741 if (auto *Extract = dyn_cast<ExtractValueInst>(Val: &Inst))
1742 return Builder.CreateFPExt(V: Extract, DestTy: ExtendedVT);
1743
1744 if (auto *BC = dyn_cast<BitCastInst>(Val: &Inst))
1745 return Builder.CreateFPExt(V: BC, DestTy: ExtendedVT);
1746
1747 report_fatal_error(reason: "Unimplemented support for " +
1748 Twine(Inst.getOpcodeName()));
1749}
1750
1751// Creates a shadow value for an instruction that defines a value of FT type.
1752// FT operands that do not already have shadow values are created recursively.
1753// The DFS is guaranteed to not loop as phis and arguments already have
1754// shadows.
1755void NumericalStabilitySanitizer::maybeCreateShadowValue(
1756 Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
1757 Type *VT = Root.getType();
1758 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1759 if (ExtendedVT == nullptr)
1760 return; // Not an FT value.
1761
1762 if (Map.hasShadow(V: &Root))
1763 return; // Shadow already exists.
1764
1765 assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");
1766
1767 std::vector<Instruction *> DfsStack(1, &Root);
1768 while (!DfsStack.empty()) {
1769 // Ensure that all operands to the instruction have shadows before
1770 // proceeding.
1771 Instruction *I = DfsStack.back();
1772 // The shadow for the instruction might have been created deeper in the DFS,
1773 // see `forward_use_with_two_uses` test.
1774 if (Map.hasShadow(V: I)) {
1775 DfsStack.pop_back();
1776 continue;
1777 }
1778
1779 bool MissingShadow = false;
1780 for (Value *Op : I->operands()) {
1781 Type *VT = Op->getType();
1782 if (!Config.getExtendedFPType(FT: VT))
1783 continue; // Not an FT value.
1784 if (Map.hasShadow(V: Op))
1785 continue; // Shadow is already available.
1786 MissingShadow = true;
1787 DfsStack.push_back(x: cast<Instruction>(Val: Op));
1788 }
1789 if (MissingShadow)
1790 continue; // Process operands and come back to this instruction later.
1791
1792 // All operands have shadows. Create a shadow for the current value.
1793 Value *Shadow = createShadowValueWithOperandsAvailable(Inst&: *I, TLI, Map);
1794 Map.setShadow(V&: *I, Shadow&: *Shadow);
1795 DfsStack.pop_back();
1796 }
1797}
1798
1799// A floating-point store needs its value and type written to shadow memory.
1800void NumericalStabilitySanitizer::propagateFTStore(
1801 StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {
1802 Value *StoredValue = Store.getValueOperand();
1803 IRBuilder<> Builder(&Store);
1804 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1805 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1806 Value *ShadowPtr = Builder.CreateCall(
1807 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1808 Args: {Store.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1809
1810 Value *StoredShadow = Map.getShadow(V: StoredValue);
1811 if (!Store.getParent()->getParent()->hasOptNone()) {
1812 // Only check stores when optimizing, because non-optimized code generates
1813 // too many stores to the stack, creating false positives.
1814 if (ClCheckStores) {
1815 StoredShadow = emitCheck(V: StoredValue, ShadowV: StoredShadow, Builder,
1816 Loc: CheckLoc::makeStore(Address: Store.getPointerOperand()));
1817 ++NumInstrumentedFTStores;
1818 }
1819 }
1820
1821 Builder.CreateAlignedStore(Val: StoredShadow, Ptr: ShadowPtr, Align: Align(1),
1822 isVolatile: Store.isVolatile());
1823}
1824
1825// A non-ft store needs to invalidate shadow memory. Exceptions are:
1826// - memory transfers of floating-point data through other pointer types (llvm
1827// optimization passes transform `*(float*)a = *(float*)b` into
1828// `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.
1829// - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted
1830// ints. Note that this is not really necessary because if the value is
1831// unknown the framework will re-extend it on load anyway. It just felt
1832// easier to debug tests with vectors of FTs.
1833void NumericalStabilitySanitizer::propagateNonFTStore(
1834 StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {
1835 Value *PtrOp = Store.getPointerOperand();
1836 IRBuilder<> Builder(Store.getNextNode());
1837 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1838 Value *Dst = PtrOp;
1839 TypeSize SlotSize = DL.getTypeStoreSize(Ty: VT);
1840 assert(!SlotSize.isScalable() && "unsupported");
1841 const auto LoadSizeBytes = SlotSize.getFixedValue();
1842 Value *ValueSize = Constant::getIntegerValue(
1843 Ty: IntptrTy, V: APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));
1844
1845 ++NumInstrumentedNonFTStores;
1846 Value *StoredValue = Store.getValueOperand();
1847 if (LoadInst *Load = dyn_cast<LoadInst>(Val: StoredValue)) {
1848 // TODO: Handle the case when the value is from a phi.
1849 // This is a memory transfer with memcpy semantics. Copy the type and
1850 // value from the source. Note that we cannot use __nsan_copy_values()
1851 // here, because that will not work when there is a write to memory in
1852 // between the load and the store, e.g. in the case of a swap.
1853 Type *ShadowTypeIntTy = Type::getIntNTy(C&: Context, N: 8 * LoadSizeBytes);
1854 Type *ShadowValueIntTy =
1855 Type::getIntNTy(C&: Context, N: 8 * kShadowScale * LoadSizeBytes);
1856 IRBuilder<> LoadBuilder(Load->getNextNode());
1857 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1858 Value *LoadSrc = Load->getPointerOperand();
1859 // Read the shadow type and value at load time. The type has the same size
1860 // as the FT value, the value has twice its size.
1861 // TODO: cache them to avoid re-creating them when a load is used by
1862 // several stores. Maybe create them like the FT shadows when a load is
1863 // encountered.
1864 Value *RawShadowType = LoadBuilder.CreateAlignedLoad(
1865 Ty: ShadowTypeIntTy,
1866 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {LoadSrc}), Align: Align(1),
1867 /*isVolatile=*/false);
1868 Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(
1869 Ty: ShadowValueIntTy,
1870 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {LoadSrc}), Align: Align(1),
1871 /*isVolatile=*/false);
1872
1873 // Write back the shadow type and value at store time.
1874 Builder.CreateAlignedStore(
1875 Val: RawShadowType, Ptr: Builder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {Dst}),
1876 Align: Align(1),
1877 /*isVolatile=*/false);
1878 Builder.CreateAlignedStore(Val: RawShadowValue,
1879 Ptr: Builder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {Dst}),
1880 Align: Align(1),
1881 /*isVolatile=*/false);
1882
1883 ++NumInstrumentedNonFTMemcpyStores;
1884 return;
1885 }
1886 // ClPropagateNonFTConstStoresAsFT is by default false.
1887 if (Constant *C; ClPropagateNonFTConstStoresAsFT &&
1888 (C = dyn_cast<Constant>(Val: StoredValue))) {
1889 // This might be a fp constant stored as an int. Bitcast and store if it has
1890 // appropriate size.
1891 Type *BitcastTy = nullptr; // The FT type to bitcast to.
1892 if (auto *CInt = dyn_cast<ConstantInt>(Val: C)) {
1893 switch (CInt->getType()->getScalarSizeInBits()) {
1894 case 32:
1895 BitcastTy = Type::getFloatTy(C&: Context);
1896 break;
1897 case 64:
1898 BitcastTy = Type::getDoubleTy(C&: Context);
1899 break;
1900 case 80:
1901 BitcastTy = Type::getX86_FP80Ty(C&: Context);
1902 break;
1903 default:
1904 break;
1905 }
1906 } else if (auto *CDV = dyn_cast<ConstantDataVector>(Val: C)) {
1907 const int NumElements =
1908 cast<VectorType>(Val: CDV->getType())->getElementCount().getFixedValue();
1909 switch (CDV->getType()->getScalarSizeInBits()) {
1910 case 32:
1911 BitcastTy =
1912 VectorType::get(ElementType: Type::getFloatTy(C&: Context), NumElements, Scalable: false);
1913 break;
1914 case 64:
1915 BitcastTy =
1916 VectorType::get(ElementType: Type::getDoubleTy(C&: Context), NumElements, Scalable: false);
1917 break;
1918 case 80:
1919 BitcastTy =
1920 VectorType::get(ElementType: Type::getX86_FP80Ty(C&: Context), NumElements, Scalable: false);
1921 break;
1922 default:
1923 break;
1924 }
1925 }
1926 if (BitcastTy) {
1927 const MemoryExtents Extents = getMemoryExtentsOrDie(FT: BitcastTy);
1928 Value *ShadowPtr = Builder.CreateCall(
1929 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1930 Args: {PtrOp, ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1931 // Bitcast the integer value to the appropriate FT type and extend to 2FT.
1932 Type *ExtVT = Config.getExtendedFPType(FT: BitcastTy);
1933 Value *Shadow =
1934 Builder.CreateFPExt(V: Builder.CreateBitCast(V: C, DestTy: BitcastTy), DestTy: ExtVT);
1935 Builder.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1),
1936 isVolatile: Store.isVolatile());
1937 return;
1938 }
1939 }
1940 // All other stores just reset the shadow value to unknown.
1941 Builder.CreateCall(Callee: NsanSetUnknownFns.getFallback(), Args: {Dst, ValueSize});
1942}
1943
1944void NumericalStabilitySanitizer::propagateShadowValues(
1945 Instruction &Inst, const TargetLibraryInfo &TLI,
1946 const ValueToShadowMap &Map) {
1947 if (auto *Store = dyn_cast<StoreInst>(Val: &Inst)) {
1948 Value *StoredValue = Store->getValueOperand();
1949 Type *VT = StoredValue->getType();
1950 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1951 if (ExtendedVT == nullptr)
1952 return propagateNonFTStore(Store&: *Store, VT, Map);
1953 return propagateFTStore(Store&: *Store, VT, ExtendedVT, Map);
1954 }
1955
1956 if (auto *FCmp = dyn_cast<FCmpInst>(Val: &Inst)) {
1957 emitFCmpCheck(FCmp&: *FCmp, Map);
1958 return;
1959 }
1960
1961 if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
1962 maybeAddSuffixForNsanInterface(CI: CB);
1963 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
1964 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
1965 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
1966 instrumentMemIntrinsic(MI);
1967 return;
1968 }
1969 populateShadowStack(CI&: *CB, TLI, Map);
1970 return;
1971 }
1972
1973 if (auto *RetInst = dyn_cast<ReturnInst>(Val: &Inst)) {
1974 if (!ClCheckRet)
1975 return;
1976
1977 Value *RV = RetInst->getReturnValue();
1978 if (RV == nullptr)
1979 return; // This is a `ret void`.
1980 Type *VT = RV->getType();
1981 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1982 if (ExtendedVT == nullptr)
1983 return; // Not an FT ret.
1984 Value *RVShadow = Map.getShadow(V: RV);
1985 IRBuilder<> Builder(RetInst);
1986
1987 RVShadow = emitCheck(V: RV, ShadowV: RVShadow, Builder, Loc: CheckLoc::makeRet());
1988 ++NumInstrumentedFTRets;
1989 // Store tag.
1990 Value *FnAddr =
1991 Builder.CreatePtrToInt(V: Inst.getParent()->getParent(), DestTy: IntptrTy);
1992 Builder.CreateStore(Val: FnAddr, Ptr: NsanShadowRetTag);
1993 // Store value.
1994 Value *ShadowRetValPtr =
1995 Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0);
1996 Builder.CreateStore(Val: RVShadow, Ptr: ShadowRetValPtr);
1997 return;
1998 }
1999
2000 if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(Val: &Inst)) {
2001 Value *V = Insert->getOperand(i_nocapture: 1);
2002 Type *VT = V->getType();
2003 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
2004 if (ExtendedVT == nullptr)
2005 return;
2006 IRBuilder<> Builder(Insert);
2007 emitCheck(V, ShadowV: Map.getShadow(V), Builder, Loc: CheckLoc::makeInsert());
2008 return;
2009 }
2010}
2011
2012// Moves fast math flags from the function to individual instructions, and
2013// removes the attribute from the function.
2014// TODO: Make this controllable with a flag.
2015static void moveFastMathFlags(Function &F,
2016 std::vector<Instruction *> &Instructions) {
2017 FastMathFlags FMF;
2018#define MOVE_FLAG(attr, setter) \
2019 if (F.getFnAttribute(attr).getValueAsString() == "true") { \
2020 F.removeFnAttr(attr); \
2021 FMF.set##setter(); \
2022 }
2023 MOVE_FLAG("no-infs-fp-math", NoInfs)
2024 MOVE_FLAG("no-nans-fp-math", NoNaNs)
2025 MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)
2026#undef MOVE_FLAG
2027
2028 for (Instruction *I : Instructions)
2029 if (isa<FPMathOperator>(Val: I))
2030 I->setFastMathFlags(FMF);
2031}
2032
2033bool NumericalStabilitySanitizer::sanitizeFunction(
2034 Function &F, const TargetLibraryInfo &TLI) {
2035 if (!F.hasFnAttribute(Kind: Attribute::SanitizeNumericalStability) ||
2036 F.isDeclaration())
2037 return false;
2038
2039 // This is required to prevent instrumenting call to __nsan_init from within
2040 // the module constructor.
2041 if (F.getName() == kNsanModuleCtorName)
2042 return false;
2043
2044 // The instrumentation maintains:
2045 // - for each IR value `v` of floating-point (or vector floating-point) type
2046 // FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.
2047 // double for float and f128 for double).
2048 // - A shadow memory, which stores `s(v)` for any `v` that has been stored,
2049 // along with a shadow memory tag, which stores whether the value in the
2050 // corresponding shadow memory is valid. Note that this might be
2051 // incorrect if a non-instrumented function stores to memory, or if
2052 // memory is stored to through a char pointer.
2053 // - A shadow stack, which holds `s(v)` for any floating-point argument `v`
2054 // of a call to an instrumented function. This allows
2055 // instrumented functions to retrieve the shadow values for their
2056 // arguments.
2057 // Because instrumented functions can be called from non-instrumented
2058 // functions, the stack needs to include a tag so that the instrumented
2059 // function knows whether shadow values are available for their
2060 // parameters (i.e. whether is was called by an instrumented function).
2061 // When shadow arguments are not available, they have to be recreated by
2062 // extending the precision of the non-shadow arguments to the non-shadow
2063 // value. Non-instrumented functions do not modify (or even know about) the
2064 // shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow
2065 // stack tag is __nsan_shadow_args_tag. The tag is any unique identifier
2066 // for the function (we use the address of the function). Both variables
2067 // are thread local.
2068 // Example:
2069 // calls shadow stack tag shadow stack
2070 // =======================================================================
2071 // non_instrumented_1() 0 0
2072 // |
2073 // v
2074 // instrumented_2(float a) 0 0
2075 // |
2076 // v
2077 // instrumented_3(float b, double c) &instrumented_3 s(b),s(c)
2078 // |
2079 // v
2080 // instrumented_4(float d) &instrumented_4 s(d)
2081 // |
2082 // v
2083 // non_instrumented_5(float e) &non_instrumented_5 s(e)
2084 // |
2085 // v
2086 // instrumented_6(float f) &non_instrumented_5 s(e)
2087 //
2088 // On entry, instrumented_2 checks whether the tag corresponds to its
2089 // function ptr.
2090 // Note that functions reset the tag to 0 after reading shadow parameters.
2091 // This ensures that the function does not erroneously read invalid data if
2092 // called twice in the same stack, once from an instrumented function and
2093 // once from an uninstrumented one. For example, in the following example,
2094 // resetting the tag in (A) ensures that (B) does not reuse the same the
2095 // shadow arguments (which would be incorrect).
2096 // instrumented_1(float a)
2097 // |
2098 // v
2099 // instrumented_2(float b) (A)
2100 // |
2101 // v
2102 // non_instrumented_3()
2103 // |
2104 // v
2105 // instrumented_2(float b) (B)
2106 //
2107 // - A shadow return slot. Any function that returns a floating-point value
2108 // places a shadow return value in __nsan_shadow_ret_val. Again, because
2109 // we might be calling non-instrumented functions, this value is guarded
2110 // by __nsan_shadow_ret_tag marker indicating which instrumented function
2111 // placed the value in __nsan_shadow_ret_val, so that the caller can check
2112 // that this corresponds to the callee. Both variables are thread local.
2113 //
2114 // For example, in the following example, the instrumentation in
2115 // `instrumented_1` rejects the shadow return value from `instrumented_3`
2116 // because is is not tagged as expected (`&instrumented_3` instead of
2117 // `non_instrumented_2`):
2118 //
2119 // instrumented_1()
2120 // |
2121 // v
2122 // float non_instrumented_2()
2123 // |
2124 // v
2125 // float instrumented_3()
2126 //
2127 // Calls of known math functions (sin, cos, exp, ...) are duplicated to call
2128 // their overload on the shadow type.
2129
2130 // Collect all instructions before processing, as creating shadow values
2131 // creates new instructions inside the function.
2132 std::vector<Instruction *> OriginalInstructions;
2133 for (BasicBlock &BB : F)
2134 for (Instruction &Inst : BB)
2135 OriginalInstructions.emplace_back(args: &Inst);
2136
2137 moveFastMathFlags(F, Instructions&: OriginalInstructions);
2138 ValueToShadowMap ValueToShadow(Config);
2139
2140 // In the first pass, we create shadow values for all FT function arguments
2141 // and all phis. This ensures that the DFS of the next pass does not have
2142 // any loops.
2143 std::vector<PHINode *> OriginalPhis;
2144 createShadowArguments(F, TLI, Map&: ValueToShadow);
2145 for (Instruction *I : OriginalInstructions) {
2146 if (PHINode *Phi = dyn_cast<PHINode>(Val: I)) {
2147 if (PHINode *Shadow = maybeCreateShadowPhi(Phi&: *Phi, TLI)) {
2148 OriginalPhis.push_back(x: Phi);
2149 ValueToShadow.setShadow(V&: *Phi, Shadow&: *Shadow);
2150 }
2151 }
2152 }
2153
2154 // Create shadow values for all instructions creating FT values.
2155 for (Instruction *I : OriginalInstructions)
2156 maybeCreateShadowValue(Root&: *I, TLI, Map&: ValueToShadow);
2157
2158 // Propagate shadow values across stores, calls and rets.
2159 for (Instruction *I : OriginalInstructions)
2160 propagateShadowValues(Inst&: *I, TLI, Map: ValueToShadow);
2161
2162 // The last pass populates shadow phis with shadow values.
2163 for (PHINode *Phi : OriginalPhis) {
2164 PHINode *ShadowPhi = cast<PHINode>(Val: ValueToShadow.getShadow(V: Phi));
2165 for (unsigned I : seq(Size: Phi->getNumOperands())) {
2166 Value *V = Phi->getOperand(i_nocapture: I);
2167 Value *Shadow = ValueToShadow.getShadow(V);
2168 BasicBlock *IncomingBB = Phi->getIncomingBlock(i: I);
2169 // For some instructions (e.g. invoke), we create the shadow in a separate
2170 // block, different from the block where the original value is created.
2171 // In that case, the shadow phi might need to refer to this block instead
2172 // of the original block.
2173 // Note that this can only happen for instructions as constant shadows are
2174 // always created in the same block.
2175 ShadowPhi->addIncoming(V: Shadow, BB: IncomingBB);
2176 }
2177 }
2178
2179 return !ValueToShadow.empty();
2180}
2181
2182static uint64_t GetMemOpSize(Value *V) {
2183 uint64_t OpSize = 0;
2184 if (Constant *C = dyn_cast<Constant>(Val: V)) {
2185 auto *CInt = dyn_cast<ConstantInt>(Val: C);
2186 if (CInt && CInt->getValue().getBitWidth() <= 64)
2187 OpSize = CInt->getValue().getZExtValue();
2188 }
2189
2190 return OpSize;
2191}
2192
2193// Instrument the memory intrinsics so that they properly modify the shadow
2194// memory.
2195bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
2196 IRBuilder<> Builder(MI);
2197 if (auto *M = dyn_cast<MemSetInst>(Val: MI)) {
2198 FunctionCallee SetUnknownFn =
2199 NsanSetUnknownFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2200 if (SetUnknownFn.getFunctionType()->getNumParams() == 1)
2201 Builder.CreateCall(Callee: SetUnknownFn, Args: {/*Address=*/M->getArgOperand(i: 0)});
2202 else
2203 Builder.CreateCall(Callee: SetUnknownFn,
2204 Args: {/*Address=*/M->getArgOperand(i: 0),
2205 /*Size=*/Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2206 DestTy: IntptrTy, isSigned: false)});
2207
2208 } else if (auto *M = dyn_cast<MemTransferInst>(Val: MI)) {
2209 FunctionCallee CopyFn =
2210 NsanCopyFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2211
2212 if (CopyFn.getFunctionType()->getNumParams() == 2)
2213 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2214 /*Source=*/M->getArgOperand(i: 1)});
2215 else
2216 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2217 /*Source=*/M->getArgOperand(i: 1),
2218 /*Size=*/
2219 Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2220 DestTy: IntptrTy, isSigned: false)});
2221 }
2222 return false;
2223}
2224
2225void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {
2226 Function *Fn = CI->getCalledFunction();
2227 if (Fn == nullptr)
2228 return;
2229
2230 if (!Fn->getName().starts_with(Prefix: "__nsan_"))
2231 return;
2232
2233 if (Fn->getName() == "__nsan_dump_shadow_mem") {
2234 assert(CI->arg_size() == 4 &&
2235 "invalid prototype for __nsan_dump_shadow_mem");
2236 // __nsan_dump_shadow_mem requires an extra parameter with the dynamic
2237 // configuration:
2238 // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)
2239 // | shadow_type_id_for_double
2240 const uint64_t shadow_value_type_ids =
2241 (static_cast<size_t>(Config.byValueType(VT: kLongDouble).getNsanTypeId())
2242 << 16) |
2243 (static_cast<size_t>(Config.byValueType(VT: kDouble).getNsanTypeId())
2244 << 8) |
2245 static_cast<size_t>(Config.byValueType(VT: kFloat).getNsanTypeId());
2246 CI->setArgOperand(i: 3, v: ConstantInt::get(Ty: IntptrTy, V: shadow_value_type_ids));
2247 }
2248}
2249