1//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the instrumentation pass for the numerical sanitizer.
10// Conceptually the pass injects shadow computations using higher precision
11// types and inserts consistency checks. For details see the paper
12// https://arxiv.org/abs/2102.12782.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
17
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Regex.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Transforms/Utils/BasicBlockUtils.h"
39#include "llvm/Transforms/Utils/Instrumentation.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43#include <cstdint>
44
45using namespace llvm;
46
47#define DEBUG_TYPE "nsan"
48
49STATISTIC(NumInstrumentedFTLoads,
50 "Number of instrumented floating-point loads");
51
52STATISTIC(NumInstrumentedFTCalls,
53 "Number of instrumented floating-point calls");
54STATISTIC(NumInstrumentedFTRets,
55 "Number of instrumented floating-point returns");
56STATISTIC(NumInstrumentedFTStores,
57 "Number of instrumented floating-point stores");
58STATISTIC(NumInstrumentedNonFTStores,
59 "Number of instrumented non floating-point stores");
60STATISTIC(
61 NumInstrumentedNonFTMemcpyStores,
62 "Number of instrumented non floating-point stores with memcpy semantics");
63STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
64
65// Using smaller shadow types types can help improve speed. For example, `dlq`
66// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to
67// `dqq`.
68static cl::opt<std::string> ClShadowMapping(
69 "nsan-shadow-type-mapping", cl::init(Val: "dqq"),
70 cl::desc("One shadow type id for each of `float`, `double`, `long double`. "
71 "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "
72 "ppc_fp128 (extended double) respectively. The default is to "
73 "shadow `float` as `double`, and `double` and `x86_fp80` as "
74 "`fp128`"),
75 cl::Hidden);
76
77static cl::opt<bool>
78 ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(Val: true),
79 cl::desc("Instrument floating-point comparisons"),
80 cl::Hidden);
81
82static cl::opt<std::string> ClCheckFunctionsFilter(
83 "check-functions-filter",
84 cl::desc("Only emit checks for arguments of functions "
85 "whose names match the given regular expression"),
86 cl::value_desc("regex"));
87
88static cl::opt<bool> ClTruncateFCmpEq(
89 "nsan-truncate-fcmp-eq", cl::init(Val: true),
90 cl::desc(
91 "This flag controls the behaviour of fcmp equality comparisons."
92 "For equality comparisons such as `x == 0.0f`, we can perform the "
93 "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "
94 " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "
95 "catch the case when `x_shadow` is accurate enough (and therefore "
96 "close enough to zero) so that `trunc(x_shadow)` is zero even though "
97 "both `x` and `x_shadow` are not"),
98 cl::Hidden);
99
100// When there is external, uninstrumented code writing to memory, the shadow
101// memory can get out of sync with the application memory. Enabling this flag
102// emits consistency checks for loads to catch this situation.
103// When everything is instrumented, this is not strictly necessary because any
104// load should have a corresponding store, but can help debug cases when the
105// framework did a bad job at tracking shadow memory modifications by failing on
106// load rather than store.
107// TODO: provide a way to resume computations from the FT value when the load
108// is inconsistent. This ensures that further computations are not polluted.
109static cl::opt<bool> ClCheckLoads("nsan-check-loads",
110 cl::desc("Check floating-point load"),
111 cl::Hidden);
112
113static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(Val: true),
114 cl::desc("Check floating-point stores"),
115 cl::Hidden);
116
117static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(Val: true),
118 cl::desc("Check floating-point return values"),
119 cl::Hidden);
120
121// LLVM may store constant floats as bitcasted ints.
122// It's not really necessary to shadow such stores,
123// if the shadow value is unknown the framework will re-extend it on load
124// anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is
125// impossible to determine the floating-point type based on the size.
126// However, for debugging purposes it can be useful to model such stores.
127static cl::opt<bool> ClPropagateNonFTConstStoresAsFT(
128 "nsan-propagate-non-ft-const-stores-as-ft",
129 cl::desc(
130 "Propagate non floating-point const stores as floating point values."
131 "For debugging purposes only"),
132 cl::Hidden);
133
134constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
135constexpr StringLiteral kNsanInitName("__nsan_init");
136
137// The following values must be kept in sync with the runtime.
138constexpr int kShadowScale = 2;
139constexpr int kMaxVectorWidth = 8;
140constexpr int kMaxNumArgs = 128;
141constexpr int kMaxShadowTypeSizeBytes = 16; // fp128
142
143namespace {
144
145// Defines the characteristics (type id, type, and floating-point semantics)
146// attached for all possible shadow types.
147class ShadowTypeConfig {
148public:
149 static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
150
151 // The LLVM Type corresponding to the shadow type.
152 virtual Type *getType(LLVMContext &Context) const = 0;
153
154 // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
155 virtual char getNsanTypeId() const = 0;
156
157 virtual ~ShadowTypeConfig() = default;
158};
159
160template <char NsanTypeId>
161class ShadowTypeConfigImpl : public ShadowTypeConfig {
162public:
163 char getNsanTypeId() const override { return NsanTypeId; }
164 static constexpr char kNsanTypeId = NsanTypeId;
165};
166
167// `double` (`d`) shadow type.
168class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
169 Type *getType(LLVMContext &Context) const override {
170 return Type::getDoubleTy(C&: Context);
171 }
172};
173
174// `x86_fp80` (`l`) shadow type: X86 long double.
175class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
176 Type *getType(LLVMContext &Context) const override {
177 return Type::getX86_FP80Ty(C&: Context);
178 }
179};
180
181// `fp128` (`q`) shadow type.
182class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
183 Type *getType(LLVMContext &Context) const override {
184 return Type::getFP128Ty(C&: Context);
185 }
186};
187
188// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
189class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
190 Type *getType(LLVMContext &Context) const override {
191 return Type::getPPC_FP128Ty(C&: Context);
192 }
193};
194
195// Creates a ShadowTypeConfig given its type id.
196std::unique_ptr<ShadowTypeConfig>
197ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
198 switch (TypeId) {
199 case F64ShadowConfig::kNsanTypeId:
200 return std::make_unique<F64ShadowConfig>();
201 case F80ShadowConfig::kNsanTypeId:
202 return std::make_unique<F80ShadowConfig>();
203 case F128ShadowConfig::kNsanTypeId:
204 return std::make_unique<F128ShadowConfig>();
205 case PPC128ShadowConfig::kNsanTypeId:
206 return std::make_unique<PPC128ShadowConfig>();
207 }
208 report_fatal_error(reason: "nsan: invalid shadow type id '" + Twine(TypeId) + "'");
209}
210
211// An enum corresponding to shadow value types. Used as indices in arrays, so
212// not an `enum class`.
213enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
214
215// If `FT` corresponds to a primitive FTValueType, return it.
216static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
217 if (FT->isFloatTy())
218 return kFloat;
219 if (FT->isDoubleTy())
220 return kDouble;
221 if (FT->isX86_FP80Ty())
222 return kLongDouble;
223 return {};
224}
225
226// Returns the LLVM type for an FTValueType.
227static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
228 switch (VT) {
229 case kFloat:
230 return Type::getFloatTy(C&: Context);
231 case kDouble:
232 return Type::getDoubleTy(C&: Context);
233 case kLongDouble:
234 return Type::getX86_FP80Ty(C&: Context);
235 case kNumValueTypes:
236 return nullptr;
237 }
238 llvm_unreachable("Unhandled FTValueType enum");
239}
240
241// Returns the type name for an FTValueType.
242static const char *typeNameFromFTValueType(FTValueType VT) {
243 switch (VT) {
244 case kFloat:
245 return "float";
246 case kDouble:
247 return "double";
248 case kLongDouble:
249 return "longdouble";
250 case kNumValueTypes:
251 return nullptr;
252 }
253 llvm_unreachable("Unhandled FTValueType enum");
254}
255
256// A specific mapping configuration of application type to shadow type for nsan
257// (see -nsan-shadow-mapping flag).
258class MappingConfig {
259public:
260 explicit MappingConfig(LLVMContext &C) : Context(C) {
261 if (ClShadowMapping.size() != 3)
262 report_fatal_error(reason: "Invalid nsan mapping: " + Twine(ClShadowMapping));
263 unsigned ShadowTypeSizeBits[kNumValueTypes];
264 for (int VT = 0; VT < kNumValueTypes; ++VT) {
265 auto Config = ShadowTypeConfig::fromNsanTypeId(TypeId: ClShadowMapping[VT]);
266 if (!Config)
267 report_fatal_error(reason: "Failed to get ShadowTypeConfig for " +
268 Twine(ClShadowMapping[VT]));
269 const unsigned AppTypeSize =
270 typeFromFTValueType(VT: static_cast<FTValueType>(VT), Context)
271 ->getScalarSizeInBits();
272 const unsigned ShadowTypeSize =
273 Config->getType(Context)->getScalarSizeInBits();
274 // Check that the shadow type size is at most kShadowScale times the
275 // application type size, so that shadow memory compoutations are valid.
276 if (ShadowTypeSize > kShadowScale * AppTypeSize)
277 report_fatal_error(reason: "Invalid nsan mapping f" + Twine(AppTypeSize) +
278 "->f" + Twine(ShadowTypeSize) +
279 ": The shadow type size should be at most " +
280 Twine(kShadowScale) +
281 " times the application type size");
282 ShadowTypeSizeBits[VT] = ShadowTypeSize;
283 Configs[VT] = std::move(Config);
284 }
285
286 // Check that the mapping is monotonous. This is required because if one
287 // does an fpextend of `float->long double` in application code, nsan is
288 // going to do an fpextend of `shadow(float) -> shadow(long double)` in
289 // shadow code. This will fail in `qql` mode, since nsan would be
290 // fpextending `f128->long`, which is invalid.
291 // TODO: Relax this.
292 if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
293 ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
294 report_fatal_error(reason: "Invalid nsan mapping: { float->f" +
295 Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
296 Twine(ShadowTypeSizeBits[kDouble]) +
297 "; long double->f" +
298 Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
299 }
300
301 const ShadowTypeConfig &byValueType(FTValueType VT) const {
302 assert(VT < FTValueType::kNumValueTypes && "invalid value type");
303 return *Configs[VT];
304 }
305
306 // Returns the extended shadow type for a given application type.
307 Type *getExtendedFPType(Type *FT) const {
308 if (const auto VT = ftValueTypeFromType(FT))
309 return Configs[*VT]->getType(Context);
310 if (FT->isVectorTy()) {
311 auto *VecTy = cast<VectorType>(Val: FT);
312 // TODO: add support for scalable vector types.
313 if (VecTy->isScalableTy())
314 return nullptr;
315 Type *ExtendedScalar = getExtendedFPType(FT: VecTy->getElementType());
316 return ExtendedScalar
317 ? VectorType::get(ElementType: ExtendedScalar, EC: VecTy->getElementCount())
318 : nullptr;
319 }
320 return nullptr;
321 }
322
323private:
324 LLVMContext &Context;
325 std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
326};
327
328// The memory extents of a type specifies how many elements of a given
329// FTValueType needs to be stored when storing this type.
330struct MemoryExtents {
331 FTValueType ValueType;
332 uint64_t NumElts;
333};
334
335static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
336 if (const auto VT = ftValueTypeFromType(FT))
337 return {.ValueType: *VT, .NumElts: 1};
338 if (auto *VecTy = dyn_cast<VectorType>(Val: FT)) {
339 const auto ScalarExtents = getMemoryExtentsOrDie(FT: VecTy->getElementType());
340 return {.ValueType: ScalarExtents.ValueType,
341 .NumElts: ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
342 }
343 llvm_unreachable("invalid value type");
344}
345
346// The location of a check. Passed as parameters to runtime checking functions.
347class CheckLoc {
348public:
349 // Creates a location that references an application memory location.
350 static CheckLoc makeStore(Value *Address) {
351 CheckLoc Result(kStore);
352 Result.Address = Address;
353 return Result;
354 }
355 static CheckLoc makeLoad(Value *Address) {
356 CheckLoc Result(kLoad);
357 Result.Address = Address;
358 return Result;
359 }
360
361 // Creates a location that references an argument, given by id.
362 static CheckLoc makeArg(int ArgId) {
363 CheckLoc Result(kArg);
364 Result.ArgId = ArgId;
365 return Result;
366 }
367
368 // Creates a location that references the return value of a function.
369 static CheckLoc makeRet() { return CheckLoc(kRet); }
370
371 // Creates a location that references a vector insert.
372 static CheckLoc makeInsert() { return CheckLoc(kInsert); }
373
374 // Returns the CheckType of location this refers to, as an integer-typed LLVM
375 // IR value.
376 Value *getType(LLVMContext &C) const {
377 return ConstantInt::get(Ty: Type::getInt32Ty(C), V: static_cast<int>(CheckTy));
378 }
379
380 // Returns a CheckType-specific value representing details of the location
381 // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
382 // IR value.
383 Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
384 switch (CheckTy) {
385 case kUnknown:
386 llvm_unreachable("unknown type");
387 case kRet:
388 case kInsert:
389 return ConstantInt::get(Ty: IntptrTy, V: 0);
390 case kArg:
391 return ConstantInt::get(Ty: IntptrTy, V: ArgId);
392 case kLoad:
393 case kStore:
394 return Builder.CreatePtrToInt(V: Address, DestTy: IntptrTy);
395 }
396 llvm_unreachable("Unhandled CheckType enum");
397 }
398
399private:
400 // Must be kept in sync with the runtime,
401 // see compiler-rt/lib/nsan/nsan_stats.h
402 enum CheckType {
403 kUnknown = 0,
404 kRet,
405 kArg,
406 kLoad,
407 kStore,
408 kInsert,
409 };
410 explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
411
412 Value *Address = nullptr;
413 const CheckType CheckTy;
414 int ArgId = -1;
415};
416
417// A map of LLVM IR values to shadow LLVM IR values.
418class ValueToShadowMap {
419public:
420 explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}
421
422 ValueToShadowMap(const ValueToShadowMap &) = delete;
423 ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;
424
425 // Sets the shadow value for a value. Asserts that the value does not already
426 // have a value.
427 void setShadow(Value &V, Value &Shadow) {
428 [[maybe_unused]] const bool Inserted = Map.try_emplace(Key: &V, Args: &Shadow).second;
429 LLVM_DEBUG({
430 if (!Inserted) {
431 if (auto *I = dyn_cast<Instruction>(&V))
432 errs() << I->getFunction()->getName() << ": ";
433 errs() << "duplicate shadow (" << &V << "): ";
434 V.dump();
435 }
436 });
437 assert(Inserted && "duplicate shadow");
438 }
439
440 // Returns true if the value already has a shadow (including if the value is a
441 // constant). If true, calling getShadow() is valid.
442 bool hasShadow(Value *V) const { return isa<Constant>(Val: V) || Map.contains(Val: V); }
443
444 // Returns the shadow value for a given value. Asserts that the value has
445 // a shadow value. Lazily creates shadows for constant values.
446 Value *getShadow(Value *V) const {
447 if (Constant *C = dyn_cast<Constant>(Val: V))
448 return getShadowConstant(C);
449 return Map.find(Val: V)->second;
450 }
451
452 bool empty() const { return Map.empty(); }
453
454private:
455 // Extends a constant application value to its shadow counterpart.
456 APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
457 bool LosesInfo = false;
458 CV.convert(ToSemantics: To, RM: APFloatBase::rmTowardZero, losesInfo: &LosesInfo);
459 return CV;
460 }
461
462 // Returns the shadow constant for the given application constant.
463 Constant *getShadowConstant(Constant *C) const {
464 if (UndefValue *U = dyn_cast<UndefValue>(Val: C)) {
465 return UndefValue::get(T: Config.getExtendedFPType(FT: U->getType()));
466 }
467 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
468 // Floating-point constants.
469 Type *Ty = Config.getExtendedFPType(FT: CFP->getType());
470 return ConstantFP::get(
471 Ty, V: extendConstantFP(CV: CFP->getValueAPF(),
472 To: Ty->getScalarType()->getFltSemantics()));
473 }
474 // Vector, array, or aggregate constants.
475 if (C->getType()->isVectorTy()) {
476 SmallVector<Constant *, 8> Elements;
477 for (int I = 0, E = cast<VectorType>(Val: C->getType())
478 ->getElementCount()
479 .getFixedValue();
480 I < E; ++I)
481 Elements.push_back(Elt: getShadowConstant(C: C->getAggregateElement(Elt: I)));
482 return ConstantVector::get(V: Elements);
483 }
484 llvm_unreachable("unimplemented");
485 }
486
487 const MappingConfig &Config;
488 DenseMap<Value *, Value *> Map;
489};
490
491class NsanMemOpFn {
492public:
493 NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,
494 size_t NumArgs);
495 FunctionCallee getFunctionFor(uint64_t MemOpSize) const;
496 FunctionCallee getFallback() const;
497
498private:
499 SmallVector<FunctionCallee> Funcs;
500 size_t NumSizedFuncs;
501};
502
503NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,
504 StringRef Fallback, size_t NumArgs) {
505 LLVMContext &Ctx = M.getContext();
506 AttributeList Attr;
507 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
508 Type *PtrTy = PointerType::getUnqual(C&: Ctx);
509 Type *VoidTy = Type::getVoidTy(C&: Ctx);
510 IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx);
511 FunctionType *SizedFnTy = nullptr;
512
513 NumSizedFuncs = Sized.size();
514
515 // First entry is fallback function
516 if (NumArgs == 3) {
517 Funcs.push_back(
518 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy));
519 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy, PtrTy}, isVarArg: false);
520 } else if (NumArgs == 2) {
521 Funcs.push_back(
522 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: IntptrTy));
523 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false);
524 } else {
525 llvm_unreachable("Unexpected value of sized functions arguments");
526 }
527
528 for (size_t i = 0; i < NumSizedFuncs; ++i)
529 Funcs.push_back(Elt: M.getOrInsertFunction(Name: Sized[i], T: SizedFnTy, AttributeList: Attr));
530}
531
532FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {
533 // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the
534 // following code assumes that the number of functions in `Func` is sufficient
535 assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");
536
537 size_t Idx =
538 MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));
539
540 return Funcs[Idx];
541}
542
543FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }
544
545/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
546/// API function declarations into the module if they don't exist already.
547/// Instantiating ensures the __nsan_init function is in the list of global
548/// constructors for the module.
549class NumericalStabilitySanitizer {
550public:
551 NumericalStabilitySanitizer(Module &M);
552 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
553
554private:
555 bool instrumentMemIntrinsic(MemIntrinsic *MI);
556 void maybeAddSuffixForNsanInterface(CallBase *CI);
557 bool addrPointsToConstantData(Value *Addr);
558 void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
559 ValueToShadowMap &Map);
560 Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
561 const TargetLibraryInfo &TLI,
562 const ValueToShadowMap &Map);
563 PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
564 void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
565 ValueToShadowMap &Map);
566
567 void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
568 const ValueToShadowMap &Map);
569
570 void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
571 const ValueToShadowMap &Map);
572 Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
573 CheckLoc Loc);
574 Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
575 CheckLoc Loc);
576 void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
577
578 // Value creation handlers.
579 Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
580 Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
581 const TargetLibraryInfo &TLI,
582 const ValueToShadowMap &Map, IRBuilder<> &Builder);
583 Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
584 const TargetLibraryInfo &TLI,
585 const ValueToShadowMap &Map,
586 IRBuilder<> &Builder);
587 Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
588 const ValueToShadowMap &Map, IRBuilder<> &Builder);
589 Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,
590 const ValueToShadowMap &Map, IRBuilder<> &Builder);
591
592 // Value propagation handlers.
593 void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
594 const ValueToShadowMap &Map);
595 void propagateNonFTStore(StoreInst &Store, Type *VT,
596 const ValueToShadowMap &Map);
597
598 const DataLayout &DL;
599 LLVMContext &Context;
600 MappingConfig Config;
601 IntegerType *IntptrTy = nullptr;
602
603 // TODO: Use std::array instead?
604 FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};
605 FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};
606 FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};
607 FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};
608
609 NsanMemOpFn NsanCopyFns;
610 NsanMemOpFn NsanSetUnknownFns;
611
612 FunctionCallee NsanGetRawShadowTypePtr;
613 FunctionCallee NsanGetRawShadowPtr;
614 GlobalValue *NsanShadowRetTag = nullptr;
615
616 Type *NsanShadowRetType = nullptr;
617 GlobalValue *NsanShadowRetPtr = nullptr;
618
619 GlobalValue *NsanShadowArgsTag = nullptr;
620
621 Type *NsanShadowArgsType = nullptr;
622 GlobalValue *NsanShadowArgsPtr = nullptr;
623
624 std::optional<Regex> CheckFunctionsFilter;
625};
626} // end anonymous namespace
627
628PreservedAnalyses
629NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
630 getOrCreateSanitizerCtorAndInitFunctions(
631 M, CtorName: kNsanModuleCtorName, InitName: kNsanInitName, /*InitArgTypes=*/{},
632 /*InitArgs=*/{},
633 // This callback is invoked when the functions are created the first
634 // time. Hook them into the global ctors list in that case:
635 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
636
637 NumericalStabilitySanitizer Nsan(M);
638 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
639 for (Function &F : M)
640 Nsan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
641
642 return PreservedAnalyses::none();
643}
644
645static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
646 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&M, Ty, Name] {
647 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
648 nullptr, Name, nullptr,
649 GlobalVariable::InitialExecTLSModel);
650 });
651}
652
653NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M)
654 : DL(M.getDataLayout()), Context(M.getContext()), Config(Context),
655 NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},
656 "__nsan_copy_values", /*NumArgs=*/3),
657 NsanSetUnknownFns(M,
658 {"__nsan_set_value_unknown_4",
659 "__nsan_set_value_unknown_8",
660 "__nsan_set_value_unknown_16"},
661 "__nsan_set_value_unknown", /*NumArgs=*/2) {
662 IntptrTy = DL.getIntPtrType(C&: Context);
663 Type *PtrTy = PointerType::getUnqual(C&: Context);
664 Type *Int32Ty = Type::getInt32Ty(C&: Context);
665 Type *Int1Ty = Type::getInt1Ty(C&: Context);
666 Type *VoidTy = Type::getVoidTy(C&: Context);
667
668 AttributeList Attr;
669 Attr = Attr.addFnAttribute(C&: Context, Kind: Attribute::NoUnwind);
670 // Initialize the runtime values (functions and global variables).
671 for (int I = 0; I < kNumValueTypes; ++I) {
672 const FTValueType VT = static_cast<FTValueType>(I);
673 const char *VTName = typeNameFromFTValueType(VT);
674 Type *VTTy = typeFromFTValueType(VT, Context);
675
676 // Load/store.
677 const std::string GetterPrefix =
678 std::string("__nsan_get_shadow_ptr_for_") + VTName;
679 NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
680 Name: GetterPrefix + "_store", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
681 NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
682 Name: GetterPrefix + "_load", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
683
684 // Check.
685 const auto &ShadowConfig = Config.byValueType(VT);
686 Type *ShadowTy = ShadowConfig.getType(Context);
687 NsanCheckValue[VT] =
688 M.getOrInsertFunction(Name: std::string("__nsan_internal_check_") + VTName +
689 "_" + ShadowConfig.getNsanTypeId(),
690 AttributeList: Attr, RetTy: Int32Ty, Args: VTTy, Args: ShadowTy, Args: Int32Ty, Args: IntptrTy);
691 NsanFCmpFail[VT] = M.getOrInsertFunction(
692 Name: std::string("__nsan_fcmp_fail_") + VTName + "_" +
693 ShadowConfig.getNsanTypeId(),
694 AttributeList: Attr, RetTy: VoidTy, Args: VTTy, Args: VTTy, Args: ShadowTy, Args: ShadowTy, Args: Int32Ty, Args: Int1Ty, Args: Int1Ty);
695 }
696
697 // TODO: Add attributes nofree, nosync, readnone, readonly,
698 NsanGetRawShadowTypePtr = M.getOrInsertFunction(
699 Name: "__nsan_internal_get_raw_shadow_type_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
700 NsanGetRawShadowPtr = M.getOrInsertFunction(
701 Name: "__nsan_internal_get_raw_shadow_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
702
703 NsanShadowRetTag = createThreadLocalGV(Name: "__nsan_shadow_ret_tag", M, Ty: IntptrTy);
704
705 NsanShadowRetType = ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
706 NumElements: kMaxVectorWidth * kMaxShadowTypeSizeBytes);
707 NsanShadowRetPtr =
708 createThreadLocalGV(Name: "__nsan_shadow_ret_ptr", M, Ty: NsanShadowRetType);
709
710 NsanShadowArgsTag =
711 createThreadLocalGV(Name: "__nsan_shadow_args_tag", M, Ty: IntptrTy);
712
713 NsanShadowArgsType =
714 ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
715 NumElements: kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
716
717 NsanShadowArgsPtr =
718 createThreadLocalGV(Name: "__nsan_shadow_args_ptr", M, Ty: NsanShadowArgsType);
719
720 if (!ClCheckFunctionsFilter.empty()) {
721 Regex R = Regex(ClCheckFunctionsFilter);
722 std::string RegexError;
723 assert(R.isValid(RegexError));
724 CheckFunctionsFilter = std::move(R);
725 }
726}
727
728// Returns true if the given LLVM Value points to constant data (typically, a
729// global variable reference).
730bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
731 // If this is a GEP, just analyze its pointer operand.
732 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Addr))
733 Addr = GEP->getPointerOperand();
734
735 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr))
736 return GV->isConstant();
737 return false;
738}
739
740// This instruments the function entry to create shadow arguments.
741// Pseudocode:
742// if (this_fn_ptr == __nsan_shadow_args_tag) {
743// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
744// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
745// ...
746// __nsan_shadow_args_tag = 0;
747// } else {
748// s(arg0) = fext(arg0);
749// s(arg1) = fext(arg1);
750// ...
751// }
752void NumericalStabilitySanitizer::createShadowArguments(
753 Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
754 assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
755
756 // Do not bother if there are no FP args.
757 if (all_of(Range: F.args(), P: [this](const Argument &Arg) {
758 return Config.getExtendedFPType(FT: Arg.getType()) == nullptr;
759 }))
760 return;
761
762 IRBuilder<> Builder(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt());
763 // The function has shadow args if the shadow args tag matches the function
764 // address.
765 Value *HasShadowArgs = Builder.CreateICmpEQ(
766 LHS: Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowArgsTag, /*isVolatile=*/false),
767 RHS: Builder.CreatePtrToInt(V: &F, DestTy: IntptrTy));
768
769 unsigned ShadowArgsOffsetBytes = 0;
770 for (Argument &Arg : F.args()) {
771 Type *VT = Arg.getType();
772 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
773 if (ExtendedVT == nullptr)
774 continue; // Not an FT value.
775 Value *L = Builder.CreateAlignedLoad(
776 Ty: ExtendedVT,
777 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
778 Idx1: ShadowArgsOffsetBytes),
779 Align: Align(1), /*isVolatile=*/false);
780 Value *Shadow = Builder.CreateSelect(C: HasShadowArgs, True: L,
781 False: Builder.CreateFPExt(V: &Arg, DestTy: ExtendedVT));
782 Map.setShadow(V&: Arg, Shadow&: *Shadow);
783 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
784 assert(!SlotSize.isScalable() && "unsupported");
785 ShadowArgsOffsetBytes += SlotSize;
786 }
787 Builder.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: 0), Ptr: NsanShadowArgsTag);
788}
789
790// Returns true if the instrumentation should emit code to check arguments
791// before a function call.
792static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
793 const std::optional<Regex> &CheckFunctionsFilter) {
794
795 Function *Fn = CI.getCalledFunction();
796
797 if (CheckFunctionsFilter) {
798 // Skip checking args of indirect calls.
799 if (Fn == nullptr)
800 return false;
801 if (CheckFunctionsFilter->match(String: Fn->getName()))
802 return true;
803 return false;
804 }
805
806 if (Fn == nullptr)
807 return true; // Always check args of indirect calls.
808
809 // Never check nsan functions, the user called them for a reason.
810 if (Fn->getName().starts_with(Prefix: "__nsan_"))
811 return false;
812
813 const auto ID = Fn->getIntrinsicID();
814 LibFunc LFunc = TLI.getLibFunc(FDecl: *Fn);
815 // Always check args of unknown functions.
816 if (ID == Intrinsic::ID() && LFunc == NotLibFunc)
817 return true;
818
819 // Do not check args of an `fabs` call that is used for a comparison.
820 // This is typically used for `fabs(a-b) < tolerance`, where what matters is
821 // the result of the comparison, which is already caught be the fcmp checks.
822 if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
823 LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
824 for (const auto &U : CI.users())
825 if (isa<CmpInst>(Val: U))
826 return false;
827
828 return true; // Default is check.
829}
830
831// Populates the shadow call stack (which contains shadow values for every
832// floating-point parameter to the function).
833void NumericalStabilitySanitizer::populateShadowStack(
834 CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
835 // Do not create a shadow stack for inline asm.
836 if (CI.isInlineAsm())
837 return;
838
839 // Do not bother if there are no FP args.
840 if (all_of(Range: CI.operands(), P: [this](const Value *Arg) {
841 return Config.getExtendedFPType(FT: Arg->getType()) == nullptr;
842 }))
843 return;
844
845 IRBuilder<> Builder(&CI);
846 SmallVector<Value *, 8> ArgShadows;
847 const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
848 for (auto [ArgIdx, Arg] : enumerate(First: CI.operands())) {
849 if (Config.getExtendedFPType(FT: Arg->getType()) == nullptr)
850 continue; // Not an FT value.
851 Value *ArgShadow = Map.getShadow(V: Arg);
852 ArgShadows.push_back(Elt: ShouldCheckArgs ? emitCheck(V: Arg, ShadowV: ArgShadow, Builder,
853 Loc: CheckLoc::makeArg(ArgId: ArgIdx))
854 : ArgShadow);
855 }
856
857 // Do not create shadow stacks for intrinsics/known lib funcs.
858 if (Function *Fn = CI.getCalledFunction()) {
859 if (Fn->isIntrinsic() || TLI.getLibFunc(FDecl: *Fn) != NotLibFunc)
860 return;
861 }
862
863 // Set the shadow stack tag.
864 Builder.CreateStore(Val: CI.getCalledOperand(), Ptr: NsanShadowArgsTag);
865 TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(ExactSize: 0);
866
867 unsigned ShadowArgId = 0;
868 for (const Value *Arg : CI.operands()) {
869 Type *VT = Arg->getType();
870 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
871 if (ExtendedVT == nullptr)
872 continue; // Not an FT value.
873 Builder.CreateAlignedStore(
874 Val: ArgShadows[ShadowArgId++],
875 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
876 Idx1: ShadowArgsOffsetBytes),
877 Align: Align(1), /*isVolatile=*/false);
878 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
879 assert(!SlotSize.isScalable() && "unsupported");
880 ShadowArgsOffsetBytes += SlotSize;
881 }
882}
883
884// Internal part of emitCheck(). Returns a value that indicates whether
885// computation should continue with the shadow or resume by re-fextending the
886// value.
887enum class ContinuationType { // Keep in sync with runtime.
888 ContinueWithShadow = 0,
889 ResumeFromValue = 1,
890};
891
892Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
893 IRBuilder<> &Builder,
894 CheckLoc Loc) {
895 // Do not emit checks for constant values, this is redundant.
896 if (isa<Constant>(Val: V))
897 return ConstantInt::get(
898 Ty: Builder.getInt32Ty(),
899 V: static_cast<int>(ContinuationType::ContinueWithShadow));
900
901 Type *Ty = V->getType();
902 if (const auto VT = ftValueTypeFromType(FT: Ty))
903 return Builder.CreateCall(
904 Callee: NsanCheckValue[*VT],
905 Args: {V, ShadowV, Loc.getType(C&: Context), Loc.getValue(IntptrTy, Builder)});
906
907 if (Ty->isVectorTy()) {
908 auto *VecTy = cast<VectorType>(Val: Ty);
909 // We currently skip scalable vector types in MappingConfig,
910 // thus we should not encounter any such types here.
911 assert(!VecTy->isScalableTy() &&
912 "Scalable vector types are not supported yet");
913 Value *CheckResult = nullptr;
914 for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
915 // We resume if any element resumes. Another option would be to create a
916 // vector shuffle with the array of ContinueWithShadow, but that is too
917 // complex.
918 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
919 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
920 Value *ComponentCheckResult =
921 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
922 CheckResult = CheckResult
923 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
924 : ComponentCheckResult;
925 }
926 return CheckResult;
927 }
928 if (Ty->isArrayTy()) {
929 Value *CheckResult = nullptr;
930 for (auto I : seq(Size: Ty->getArrayNumElements())) {
931 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
932 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
933 Value *ComponentCheckResult =
934 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
935 CheckResult = CheckResult
936 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
937 : ComponentCheckResult;
938 }
939 return CheckResult;
940 }
941 if (Ty->isStructTy()) {
942 Value *CheckResult = nullptr;
943 for (auto I : seq(Size: Ty->getStructNumElements())) {
944 if (Config.getExtendedFPType(FT: Ty->getStructElementType(N: I)) == nullptr)
945 continue; // Only check FT values.
946 Value *ExtractV = Builder.CreateExtractValue(Agg: V, Idxs: I);
947 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
948 Value *ComponentCheckResult =
949 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
950 CheckResult = CheckResult
951 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
952 : ComponentCheckResult;
953 }
954 if (!CheckResult)
955 return ConstantInt::get(
956 Ty: Builder.getInt32Ty(),
957 V: static_cast<int>(ContinuationType::ContinueWithShadow));
958 return CheckResult;
959 }
960
961 llvm_unreachable("not implemented");
962}
963
964// Inserts a runtime check of V against its shadow value ShadowV.
965// We check values whenever they escape: on return, call, stores, and
966// insertvalue.
967// Returns the shadow value that should be used to continue the computations,
968// depending on the answer from the runtime.
969// TODO: Should we check on select ? phi ?
970Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
971 IRBuilder<> &Builder,
972 CheckLoc Loc) {
973 // Do not emit checks for constant values, this is redundant.
974 if (isa<Constant>(Val: V))
975 return ShadowV;
976
977 if (Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
978 Function *F = Inst->getFunction();
979 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName())) {
980 return ShadowV;
981 }
982 }
983
984 Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
985 Value *ICmpEQ = Builder.CreateICmpEQ(
986 LHS: CheckResult,
987 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
988 V: static_cast<int>(ContinuationType::ResumeFromValue)));
989 return Builder.CreateSelect(
990 C: ICmpEQ, True: Builder.CreateFPExt(V, DestTy: Config.getExtendedFPType(FT: V->getType())),
991 False: ShadowV);
992}
993
994// Inserts a check that fcmp on shadow values are consistent with that on base
995// values.
996void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
997 const ValueToShadowMap &Map) {
998 if (!ClInstrumentFCmp)
999 return;
1000
1001 Function *F = FCmp.getFunction();
1002 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName()))
1003 return;
1004
1005 Value *LHS = FCmp.getOperand(i_nocapture: 0);
1006 if (Config.getExtendedFPType(FT: LHS->getType()) == nullptr)
1007 return;
1008 Value *RHS = FCmp.getOperand(i_nocapture: 1);
1009
1010 // Split the basic block. On mismatch, we'll jump to the new basic block with
1011 // a call to the runtime for error reporting.
1012 BasicBlock *FCmpBB = FCmp.getParent();
1013 BasicBlock *NextBB = FCmpBB->splitBasicBlock(I: FCmp.getNextNode());
1014 // Remove the newly created terminator unconditional branch.
1015 FCmpBB->back().eraseFromParent();
1016 BasicBlock *FailBB =
1017 BasicBlock::Create(Context, Name: "", Parent: FCmpBB->getParent(), InsertBefore: NextBB);
1018
1019 // Create the shadow fcmp and comparison between the fcmps.
1020 IRBuilder<> FCmpBuilder(FCmpBB);
1021 FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1022 Value *ShadowLHS = Map.getShadow(V: LHS);
1023 Value *ShadowRHS = Map.getShadow(V: RHS);
1024 // See comment on ClTruncateFCmpEq.
1025 if (FCmp.isEquality() && ClTruncateFCmpEq) {
1026 Type *Ty = ShadowLHS->getType();
1027 ShadowLHS = FCmpBuilder.CreateFPExt(
1028 V: FCmpBuilder.CreateFPTrunc(V: ShadowLHS, DestTy: LHS->getType()), DestTy: Ty);
1029 ShadowRHS = FCmpBuilder.CreateFPExt(
1030 V: FCmpBuilder.CreateFPTrunc(V: ShadowRHS, DestTy: RHS->getType()), DestTy: Ty);
1031 }
1032 Value *ShadowFCmp =
1033 FCmpBuilder.CreateFCmp(P: FCmp.getPredicate(), LHS: ShadowLHS, RHS: ShadowRHS);
1034 Value *OriginalAndShadowFcmpMatch =
1035 FCmpBuilder.CreateICmpEQ(LHS: &FCmp, RHS: ShadowFCmp);
1036
1037 if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
1038 // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
1039 // where an element is true if the corresponding elements in original and
1040 // shadow are the same. We want all elements to be 1.
1041 OriginalAndShadowFcmpMatch =
1042 FCmpBuilder.CreateAndReduce(Src: OriginalAndShadowFcmpMatch);
1043 }
1044
1045 // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common
1046 // case.
1047 FCmpBuilder.CreateCondBr(Cond: OriginalAndShadowFcmpMatch, True: NextBB, False: FailBB,
1048 BranchWeights: MDBuilder(Context).createLikelyBranchWeights());
1049
1050 // Fill in FailBB.
1051 IRBuilder<> FailBuilder(FailBB);
1052 FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1053
1054 const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
1055 &FailBuilder](Value *L, Value *R, Value *ShadowL,
1056 Value *ShadowR, Value *Result,
1057 Value *ShadowResult) {
1058 Type *FT = L->getType();
1059 FunctionCallee *Callee = nullptr;
1060 if (FT->isFloatTy()) {
1061 Callee = &(NsanFCmpFail[kFloat]);
1062 } else if (FT->isDoubleTy()) {
1063 Callee = &(NsanFCmpFail[kDouble]);
1064 } else if (FT->isX86_FP80Ty()) {
1065 // TODO: make NsanFCmpFailLongDouble work.
1066 Callee = &(NsanFCmpFail[kDouble]);
1067 L = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1068 R = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1069 } else {
1070 llvm_unreachable("not implemented");
1071 }
1072 FailBuilder.CreateCall(Callee: *Callee, Args: {L, R, ShadowL, ShadowR,
1073 ConstantInt::get(Ty: FCmpBuilder.getInt32Ty(),
1074 V: FCmp.getPredicate()),
1075 Result, ShadowResult});
1076 };
1077 if (LHS->getType()->isVectorTy()) {
1078 for (int I = 0, E = cast<VectorType>(Val: LHS->getType())
1079 ->getElementCount()
1080 .getFixedValue();
1081 I < E; ++I) {
1082 Value *ExtractLHS = FailBuilder.CreateExtractElement(Vec: LHS, Idx: I);
1083 Value *ExtractRHS = FailBuilder.CreateExtractElement(Vec: RHS, Idx: I);
1084 Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(Vec: ShadowLHS, Idx: I);
1085 Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(Vec: ShadowRHS, Idx: I);
1086 Value *ExtractFCmp = FailBuilder.CreateExtractElement(Vec: &FCmp, Idx: I);
1087 Value *ExtractShadowFCmp =
1088 FailBuilder.CreateExtractElement(Vec: ShadowFCmp, Idx: I);
1089 EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,
1090 ExtractFCmp, ExtractShadowFCmp);
1091 }
1092 } else {
1093 EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
1094 }
1095 FailBuilder.CreateBr(Dest: NextBB);
1096
1097 ++NumInstrumentedFCmp;
1098}
1099
1100// Creates a shadow phi value for any phi that defines a value of FT type.
1101PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
1102 PHINode &Phi, const TargetLibraryInfo &TLI) {
1103 Type *VT = Phi.getType();
1104 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1105 if (ExtendedVT == nullptr)
1106 return nullptr; // Not an FT value.
1107 // The phi operands are shadow values and are not available when the phi is
1108 // created. They will be populated in a final phase, once all shadow values
1109 // have been created.
1110 PHINode *Shadow = PHINode::Create(Ty: ExtendedVT, NumReservedValues: Phi.getNumIncomingValues());
1111 Shadow->insertAfter(InsertPos: Phi.getIterator());
1112 return Shadow;
1113}
1114
1115Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
1116 Type *ExtendedVT) {
1117 IRBuilder<> Builder(Load.getNextNode());
1118 Builder.SetCurrentDebugLocation(Load.getDebugLoc());
1119 if (addrPointsToConstantData(Addr: Load.getPointerOperand())) {
1120 // No need to look into the shadow memory, the value is a constant. Just
1121 // convert from FT to 2FT.
1122 return Builder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1123 }
1124
1125 // if (%shadowptr == &)
1126 // %shadow = fpext %v
1127 // else
1128 // %shadow = load (ptrcast %shadow_ptr))
1129 // Considered options here:
1130 // - Have `NsanGetShadowPtrForLoad` return a fixed address
1131 // &__nsan_unknown_value_shadow_address that is valid to load from, and
1132 // use a select. This has the advantage that the generated IR is simpler.
1133 // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does
1134 // not short-circuit, dereferencing the returned pointer is no longer an
1135 // option, have to split and create a separate basic block. This has the
1136 // advantage of being easier to debug because it crashes if we ever mess
1137 // up.
1138
1139 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1140 Value *ShadowPtr = Builder.CreateCall(
1141 Callee: NsanGetShadowPtrForLoad[Extents.ValueType],
1142 Args: {Load.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1143 ++NumInstrumentedFTLoads;
1144
1145 // Split the basic block.
1146 BasicBlock *LoadBB = Load.getParent();
1147 BasicBlock *NextBB = LoadBB->splitBasicBlock(I: Builder.GetInsertPoint());
1148 // Create the two options for creating the shadow value.
1149 BasicBlock *ShadowLoadBB =
1150 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1151 BasicBlock *FExtBB =
1152 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1153
1154 // Replace the newly created terminator unconditional branch by a conditional
1155 // branch to one of the options.
1156 {
1157 LoadBB->back().eraseFromParent();
1158 IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
1159 LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1160 LoadBBBuilder.CreateCondBr(Cond: LoadBBBuilder.CreateIsNull(Arg: ShadowPtr), True: FExtBB,
1161 False: ShadowLoadBB);
1162 }
1163
1164 // Fill in ShadowLoadBB.
1165 IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
1166 ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1167 Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
1168 Ty: ExtendedVT, Ptr: ShadowPtr, Align: Align(1), isVolatile: Load.isVolatile());
1169 if (ClCheckLoads) {
1170 ShadowLoad = emitCheck(V: &Load, ShadowV: ShadowLoad, Builder&: ShadowLoadBBBuilder,
1171 Loc: CheckLoc::makeLoad(Address: Load.getPointerOperand()));
1172 }
1173 ShadowLoadBBBuilder.CreateBr(Dest: NextBB);
1174
1175 // Fill in FExtBB.
1176 IRBuilder<> FExtBBBuilder(FExtBB);
1177 FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1178 Value *FExt = FExtBBBuilder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1179 FExtBBBuilder.CreateBr(Dest: NextBB);
1180
1181 // The shadow value come from any of the options.
1182 IRBuilder<> NextBBBuilder(&*NextBB->begin());
1183 NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1184 PHINode *ShadowPhi = NextBBBuilder.CreatePHI(Ty: ExtendedVT, NumReservedValues: 2);
1185 ShadowPhi->addIncoming(V: ShadowLoad, BB: ShadowLoadBB);
1186 ShadowPhi->addIncoming(V: FExt, BB: FExtBB);
1187 return ShadowPhi;
1188}
1189
1190Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,
1191 Type *VT, Type *ExtendedVT,
1192 const ValueToShadowMap &Map,
1193 IRBuilder<> &Builder) {
1194 Value *OrigSource = Trunc.getOperand(i_nocapture: 0);
1195 Type *OrigSourceTy = OrigSource->getType();
1196 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1197
1198 // When truncating:
1199 // - (A) If the source has a shadow, we truncate from the shadow, else we
1200 // truncate from the original source.
1201 // - (B) If the shadow of the source is larger than the shadow of the dest,
1202 // we still need a truncate. Else, the shadow of the source is the same
1203 // type as the shadow of the dest (because mappings are non-decreasing), so
1204 // we don't need to emit a truncate.
1205 // Examples,
1206 // with a mapping of {f32->f64;f64->f80;f80->f128}
1207 // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double
1208 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1209 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1210 // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1)
1211 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80
1212 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1213 // with a mapping of {f32->f64;f64->f128;f80->f128}
1214 // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double
1215 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1216 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1217 // fptrunc x86_fp80 %1 to double -> fp128 %1
1218 // fptrunc fp128 %1 to double -> fp128 %1
1219 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1220 // with a mapping of {f32->f32;f64->f32;f80->f64}
1221 // fptrunc double %1 to float -> float s(%1)
1222 // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float
1223 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float
1224 // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float
1225 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float
1226 // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double
1227
1228 // See (A) above.
1229 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1230 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1231 // See (B) above.
1232 if (SourceTy == ExtendedVT)
1233 return Source;
1234
1235 return Builder.CreateFPTrunc(V: Source, DestTy: ExtendedVT);
1236}
1237
1238Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,
1239 Type *ExtendedVT,
1240 const ValueToShadowMap &Map,
1241 IRBuilder<> &Builder) {
1242 Value *OrigSource = Ext.getOperand(i_nocapture: 0);
1243 Type *OrigSourceTy = OrigSource->getType();
1244 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1245 // When extending:
1246 // - (A) If the source has a shadow, we extend from the shadow, else we
1247 // extend from the original source.
1248 // - (B) If the shadow of the dest is larger than the shadow of the source,
1249 // we still need an extend. Else, the shadow of the source is the same
1250 // type as the shadow of the dest (because mappings are non-decreasing), so
1251 // we don't need to emit an extend.
1252 // Examples,
1253 // with a mapping of {f32->f64;f64->f80;f80->f128}
1254 // fpext half %1 to float -> fpext half %1 to double
1255 // fpext half %1 to double -> fpext half %1 to x86_fp80
1256 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1257 // fpext float %1 to double -> double s(%1)
1258 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1259 // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128
1260 // with a mapping of {f32->f64;f64->f128;f80->f128}
1261 // fpext half %1 to float -> fpext half %1 to double
1262 // fpext half %1 to double -> fpext half %1 to fp128
1263 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1264 // fpext float %1 to double -> fpext double s(%1) to fp128
1265 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1266 // fpext double %1 to x86_fp80 -> fp128 s(%1)
1267 // with a mapping of {f32->f32;f64->f32;f80->f64}
1268 // fpext half %1 to float -> fpext half %1 to float
1269 // fpext half %1 to double -> fpext half %1 to float
1270 // fpext half %1 to x86_fp80 -> fpext half %1 to double
1271 // fpext float %1 to double -> s(%1)
1272 // fpext float %1 to x86_fp80 -> fpext float s(%1) to double
1273 // fpext double %1 to x86_fp80 -> fpext float s(%1) to double
1274
1275 // See (A) above.
1276 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1277 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1278 // See (B) above.
1279 if (SourceTy == ExtendedVT)
1280 return Source;
1281
1282 return Builder.CreateFPExt(V: Source, DestTy: ExtendedVT);
1283}
1284
1285namespace {
1286// TODO: This should be tablegen-ed.
1287struct KnownIntrinsic {
1288 struct WidenedIntrinsic {
1289 const char *NarrowName;
1290 Intrinsic::ID ID; // wide id.
1291 using FnTypeFactory = FunctionType *(*)(LLVMContext &);
1292 FnTypeFactory MakeFnTy;
1293 };
1294
1295 static const char *get(LibFunc LFunc);
1296
1297 // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
1298 // that applies the same operation on the shadow argument.
1299 // Options are:
1300 // - pass in the ID and full function type,
1301 // - pass in the name, which includes the function type through mangling.
1302 static const WidenedIntrinsic *widen(StringRef Name);
1303
1304private:
1305 struct LFEntry {
1306 LibFunc LFunc;
1307 const char *IntrinsicName;
1308 };
1309 static const LFEntry kLibfuncIntrinsics[];
1310
1311 static const WidenedIntrinsic kWidenedIntrinsics[];
1312};
1313} // namespace
1314
1315static FunctionType *makeDoubleDouble(LLVMContext &C) {
1316 return FunctionType::get(Result: Type::getDoubleTy(C), Params: {Type::getDoubleTy(C)}, isVarArg: false);
1317}
1318
1319static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {
1320 return FunctionType::get(Result: Type::getX86_FP80Ty(C), Params: {Type::getX86_FP80Ty(C)},
1321 isVarArg: false);
1322}
1323
1324static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {
1325 return FunctionType::get(Result: Type::getDoubleTy(C),
1326 Params: {Type::getDoubleTy(C), Type::getInt32Ty(C)}, isVarArg: false);
1327}
1328
1329static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {
1330 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1331 Params: {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
1332 isVarArg: false);
1333}
1334
1335static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {
1336 return FunctionType::get(Result: Type::getDoubleTy(C),
1337 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C)}, isVarArg: false);
1338}
1339
1340static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {
1341 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1342 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1343 isVarArg: false);
1344}
1345
1346static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {
1347 return FunctionType::get(
1348 Result: Type::getDoubleTy(C),
1349 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
1350 isVarArg: false);
1351}
1352
1353static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {
1354 return FunctionType::get(
1355 Result: Type::getX86_FP80Ty(C),
1356 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1357 isVarArg: false);
1358}
1359
1360const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
1361 // TODO: Right now we ignore vector intrinsics.
1362 // This is hard because we have to model the semantics of the intrinsics,
1363 // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
1364 // Intrinsics that take any non-vector FT types:
1365 // NOTE: Right now because of
1366 // https://github.com/llvm/llvm-project/issues/44744
1367 // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and
1368 // come back).
1369 {.NarrowName: "llvm.sqrt.f32", .ID: Intrinsic::sqrt, .MakeFnTy: makeDoubleDouble},
1370 {.NarrowName: "llvm.sqrt.f64", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1371 {.NarrowName: "llvm.sqrt.f80", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1372 {.NarrowName: "llvm.powi.f32", .ID: Intrinsic::powi, .MakeFnTy: makeDoubleDoubleI32},
1373 {.NarrowName: "llvm.powi.f64", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1374 {.NarrowName: "llvm.powi.f80", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1375 {.NarrowName: "llvm.sin.f32", .ID: Intrinsic::sin, .MakeFnTy: makeDoubleDouble},
1376 {.NarrowName: "llvm.sin.f64", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1377 {.NarrowName: "llvm.sin.f80", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1378 {.NarrowName: "llvm.cos.f32", .ID: Intrinsic::cos, .MakeFnTy: makeDoubleDouble},
1379 {.NarrowName: "llvm.cos.f64", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1380 {.NarrowName: "llvm.cos.f80", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1381 {.NarrowName: "llvm.pow.f32", .ID: Intrinsic::pow, .MakeFnTy: makeDoubleDoubleDouble},
1382 {.NarrowName: "llvm.pow.f64", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1383 {.NarrowName: "llvm.pow.f80", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1384 {.NarrowName: "llvm.exp.f32", .ID: Intrinsic::exp, .MakeFnTy: makeDoubleDouble},
1385 {.NarrowName: "llvm.exp.f64", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1386 {.NarrowName: "llvm.exp.f80", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1387 {.NarrowName: "llvm.exp2.f32", .ID: Intrinsic::exp2, .MakeFnTy: makeDoubleDouble},
1388 {.NarrowName: "llvm.exp2.f64", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1389 {.NarrowName: "llvm.exp2.f80", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1390 {.NarrowName: "llvm.log.f32", .ID: Intrinsic::log, .MakeFnTy: makeDoubleDouble},
1391 {.NarrowName: "llvm.log.f64", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1392 {.NarrowName: "llvm.log.f80", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1393 {.NarrowName: "llvm.log10.f32", .ID: Intrinsic::log10, .MakeFnTy: makeDoubleDouble},
1394 {.NarrowName: "llvm.log10.f64", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1395 {.NarrowName: "llvm.log10.f80", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1396 {.NarrowName: "llvm.log2.f32", .ID: Intrinsic::log2, .MakeFnTy: makeDoubleDouble},
1397 {.NarrowName: "llvm.log2.f64", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1398 {.NarrowName: "llvm.log2.f80", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1399 {.NarrowName: "llvm.fma.f32", .ID: Intrinsic::fma, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1400 {.NarrowName: "llvm.fma.f64", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1401 {.NarrowName: "llvm.fma.f80", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1402 {.NarrowName: "llvm.fmuladd.f32", .ID: Intrinsic::fmuladd, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1403 {.NarrowName: "llvm.fmuladd.f64", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1404 {.NarrowName: "llvm.fmuladd.f80", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1405 {.NarrowName: "llvm.fabs.f32", .ID: Intrinsic::fabs, .MakeFnTy: makeDoubleDouble},
1406 {.NarrowName: "llvm.fabs.f64", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1407 {.NarrowName: "llvm.fabs.f80", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1408 {.NarrowName: "llvm.minnum.f32", .ID: Intrinsic::minnum, .MakeFnTy: makeDoubleDoubleDouble},
1409 {.NarrowName: "llvm.minnum.f64", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1410 {.NarrowName: "llvm.minnum.f80", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1411 {.NarrowName: "llvm.maxnum.f32", .ID: Intrinsic::maxnum, .MakeFnTy: makeDoubleDoubleDouble},
1412 {.NarrowName: "llvm.maxnum.f64", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1413 {.NarrowName: "llvm.maxnum.f80", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1414 {.NarrowName: "llvm.minimum.f32", .ID: Intrinsic::minimum, .MakeFnTy: makeDoubleDoubleDouble},
1415 {.NarrowName: "llvm.minimum.f64", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1416 {.NarrowName: "llvm.minimum.f80", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1417 {.NarrowName: "llvm.maximum.f32", .ID: Intrinsic::maximum, .MakeFnTy: makeDoubleDoubleDouble},
1418 {.NarrowName: "llvm.maximum.f64", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1419 {.NarrowName: "llvm.maximum.f80", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1420 {.NarrowName: "llvm.minimumnum.f32", .ID: Intrinsic::minimumnum, .MakeFnTy: makeDoubleDoubleDouble},
1421 {.NarrowName: "llvm.minimumnum.f64", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1422 {.NarrowName: "llvm.minimumnum.f80", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1423 {.NarrowName: "llvm.maximumnum.f32", .ID: Intrinsic::maximumnum, .MakeFnTy: makeDoubleDoubleDouble},
1424 {.NarrowName: "llvm.maximumnum.f64", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1425 {.NarrowName: "llvm.maximumnum.f80", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1426 {.NarrowName: "llvm.copysign.f32", .ID: Intrinsic::copysign, .MakeFnTy: makeDoubleDoubleDouble},
1427 {.NarrowName: "llvm.copysign.f64", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1428 {.NarrowName: "llvm.copysign.f80", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1429 {.NarrowName: "llvm.floor.f32", .ID: Intrinsic::floor, .MakeFnTy: makeDoubleDouble},
1430 {.NarrowName: "llvm.floor.f64", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1431 {.NarrowName: "llvm.floor.f80", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1432 {.NarrowName: "llvm.ceil.f32", .ID: Intrinsic::ceil, .MakeFnTy: makeDoubleDouble},
1433 {.NarrowName: "llvm.ceil.f64", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1434 {.NarrowName: "llvm.ceil.f80", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1435 {.NarrowName: "llvm.trunc.f32", .ID: Intrinsic::trunc, .MakeFnTy: makeDoubleDouble},
1436 {.NarrowName: "llvm.trunc.f64", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1437 {.NarrowName: "llvm.trunc.f80", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1438 {.NarrowName: "llvm.rint.f32", .ID: Intrinsic::rint, .MakeFnTy: makeDoubleDouble},
1439 {.NarrowName: "llvm.rint.f64", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1440 {.NarrowName: "llvm.rint.f80", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1441 {.NarrowName: "llvm.nearbyint.f32", .ID: Intrinsic::nearbyint, .MakeFnTy: makeDoubleDouble},
1442 {.NarrowName: "llvm.nearbyint.f64", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1443 {.NarrowName: "llvm.nearbyint.f80", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1444 {.NarrowName: "llvm.round.f32", .ID: Intrinsic::round, .MakeFnTy: makeDoubleDouble},
1445 {.NarrowName: "llvm.round.f64", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1446 {.NarrowName: "llvm.round.f80", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1447};
1448
1449const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {
1450 {.LFunc: LibFunc_sqrtf, .IntrinsicName: "llvm.sqrt.f32"},
1451 {.LFunc: LibFunc_sqrt, .IntrinsicName: "llvm.sqrt.f64"},
1452 {.LFunc: LibFunc_sqrtl, .IntrinsicName: "llvm.sqrt.f80"},
1453 {.LFunc: LibFunc_sinf, .IntrinsicName: "llvm.sin.f32"},
1454 {.LFunc: LibFunc_sin, .IntrinsicName: "llvm.sin.f64"},
1455 {.LFunc: LibFunc_sinl, .IntrinsicName: "llvm.sin.f80"},
1456 {.LFunc: LibFunc_cosf, .IntrinsicName: "llvm.cos.f32"},
1457 {.LFunc: LibFunc_cos, .IntrinsicName: "llvm.cos.f64"},
1458 {.LFunc: LibFunc_cosl, .IntrinsicName: "llvm.cos.f80"},
1459 {.LFunc: LibFunc_powf, .IntrinsicName: "llvm.pow.f32"},
1460 {.LFunc: LibFunc_pow, .IntrinsicName: "llvm.pow.f64"},
1461 {.LFunc: LibFunc_powl, .IntrinsicName: "llvm.pow.f80"},
1462 {.LFunc: LibFunc_expf, .IntrinsicName: "llvm.exp.f32"},
1463 {.LFunc: LibFunc_exp, .IntrinsicName: "llvm.exp.f64"},
1464 {.LFunc: LibFunc_expl, .IntrinsicName: "llvm.exp.f80"},
1465 {.LFunc: LibFunc_exp2f, .IntrinsicName: "llvm.exp2.f32"},
1466 {.LFunc: LibFunc_exp2, .IntrinsicName: "llvm.exp2.f64"},
1467 {.LFunc: LibFunc_exp2l, .IntrinsicName: "llvm.exp2.f80"},
1468 {.LFunc: LibFunc_logf, .IntrinsicName: "llvm.log.f32"},
1469 {.LFunc: LibFunc_log, .IntrinsicName: "llvm.log.f64"},
1470 {.LFunc: LibFunc_logl, .IntrinsicName: "llvm.log.f80"},
1471 {.LFunc: LibFunc_log10f, .IntrinsicName: "llvm.log10.f32"},
1472 {.LFunc: LibFunc_log10, .IntrinsicName: "llvm.log10.f64"},
1473 {.LFunc: LibFunc_log10l, .IntrinsicName: "llvm.log10.f80"},
1474 {.LFunc: LibFunc_log2f, .IntrinsicName: "llvm.log2.f32"},
1475 {.LFunc: LibFunc_log2, .IntrinsicName: "llvm.log2.f64"},
1476 {.LFunc: LibFunc_log2l, .IntrinsicName: "llvm.log2.f80"},
1477 {.LFunc: LibFunc_fabsf, .IntrinsicName: "llvm.fabs.f32"},
1478 {.LFunc: LibFunc_fabs, .IntrinsicName: "llvm.fabs.f64"},
1479 {.LFunc: LibFunc_fabsl, .IntrinsicName: "llvm.fabs.f80"},
1480 {.LFunc: LibFunc_copysignf, .IntrinsicName: "llvm.copysign.f32"},
1481 {.LFunc: LibFunc_copysign, .IntrinsicName: "llvm.copysign.f64"},
1482 {.LFunc: LibFunc_copysignl, .IntrinsicName: "llvm.copysign.f80"},
1483 {.LFunc: LibFunc_floorf, .IntrinsicName: "llvm.floor.f32"},
1484 {.LFunc: LibFunc_floor, .IntrinsicName: "llvm.floor.f64"},
1485 {.LFunc: LibFunc_floorl, .IntrinsicName: "llvm.floor.f80"},
1486 {.LFunc: LibFunc_fmaxf, .IntrinsicName: "llvm.maxnum.f32"},
1487 {.LFunc: LibFunc_fmax, .IntrinsicName: "llvm.maxnum.f64"},
1488 {.LFunc: LibFunc_fmaxl, .IntrinsicName: "llvm.maxnum.f80"},
1489 {.LFunc: LibFunc_fminf, .IntrinsicName: "llvm.minnum.f32"},
1490 {.LFunc: LibFunc_fmin, .IntrinsicName: "llvm.minnum.f64"},
1491 {.LFunc: LibFunc_fminl, .IntrinsicName: "llvm.minnum.f80"},
1492 {.LFunc: LibFunc_fmaximum_numf, .IntrinsicName: "llvm.maximumnum.f32"},
1493 {.LFunc: LibFunc_fmaximum_num, .IntrinsicName: "llvm.maximumnum.f64"},
1494 {.LFunc: LibFunc_fmaximum_numl, .IntrinsicName: "llvm.maximumnum.f80"},
1495 {.LFunc: LibFunc_fminimum_numf, .IntrinsicName: "llvm.minimumnum.f32"},
1496 {.LFunc: LibFunc_fminimum_num, .IntrinsicName: "llvm.minimumnum.f64"},
1497 {.LFunc: LibFunc_fminimum_numl, .IntrinsicName: "llvm.minimumnum.f80"},
1498 {.LFunc: LibFunc_ceilf, .IntrinsicName: "llvm.ceil.f32"},
1499 {.LFunc: LibFunc_ceil, .IntrinsicName: "llvm.ceil.f64"},
1500 {.LFunc: LibFunc_ceill, .IntrinsicName: "llvm.ceil.f80"},
1501 {.LFunc: LibFunc_truncf, .IntrinsicName: "llvm.trunc.f32"},
1502 {.LFunc: LibFunc_trunc, .IntrinsicName: "llvm.trunc.f64"},
1503 {.LFunc: LibFunc_truncl, .IntrinsicName: "llvm.trunc.f80"},
1504 {.LFunc: LibFunc_rintf, .IntrinsicName: "llvm.rint.f32"},
1505 {.LFunc: LibFunc_rint, .IntrinsicName: "llvm.rint.f64"},
1506 {.LFunc: LibFunc_rintl, .IntrinsicName: "llvm.rint.f80"},
1507 {.LFunc: LibFunc_nearbyintf, .IntrinsicName: "llvm.nearbyint.f32"},
1508 {.LFunc: LibFunc_nearbyint, .IntrinsicName: "llvm.nearbyint.f64"},
1509 {.LFunc: LibFunc_nearbyintl, .IntrinsicName: "llvm.nearbyint.f80"},
1510 {.LFunc: LibFunc_roundf, .IntrinsicName: "llvm.round.f32"},
1511 {.LFunc: LibFunc_round, .IntrinsicName: "llvm.round.f64"},
1512 {.LFunc: LibFunc_roundl, .IntrinsicName: "llvm.round.f80"},
1513};
1514
1515const char *KnownIntrinsic::get(LibFunc LFunc) {
1516 for (const auto &E : kLibfuncIntrinsics) {
1517 if (E.LFunc == LFunc)
1518 return E.IntrinsicName;
1519 }
1520 return nullptr;
1521}
1522
1523const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {
1524 for (const auto &E : kWidenedIntrinsics) {
1525 if (E.NarrowName == Name)
1526 return &E;
1527 }
1528 return nullptr;
1529}
1530
1531// Returns the name of the LLVM intrinsic corresponding to the given function.
1532static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,
1533 const TargetLibraryInfo &TLI) {
1534 LibFunc LFunc = TLI.getLibFunc(FDecl: Fn);
1535 if (LFunc == NotLibFunc)
1536 return nullptr;
1537
1538 if (const char *Name = KnownIntrinsic::get(LFunc))
1539 return Name;
1540
1541 LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");
1542 return nullptr;
1543}
1544
1545// Try to handle a known function call.
1546Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(
1547 CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,
1548 const ValueToShadowMap &Map, IRBuilder<> &Builder) {
1549 Function *Fn = Call.getCalledFunction();
1550 if (Fn == nullptr)
1551 return nullptr;
1552
1553 Intrinsic::ID WidenedId = Intrinsic::ID();
1554 FunctionType *WidenedFnTy = nullptr;
1555 if (const auto ID = Fn->getIntrinsicID()) {
1556 const auto *Widened = KnownIntrinsic::widen(Name: Fn->getName());
1557 if (Widened) {
1558 WidenedId = Widened->ID;
1559 WidenedFnTy = Widened->MakeFnTy(Context);
1560 } else {
1561 // If we don't know how to widen the intrinsic, we have no choice but to
1562 // call the non-wide version on a truncated shadow and extend again
1563 // afterwards.
1564 WidenedId = ID;
1565 WidenedFnTy = Fn->getFunctionType();
1566 }
1567 } else if (const char *Name = getIntrinsicFromLibfunc(Fn&: *Fn, VT, TLI)) {
1568 // We might have a call to a library function that we can replace with a
1569 // wider Intrinsic.
1570 const auto *Widened = KnownIntrinsic::widen(Name);
1571 assert(Widened && "make sure KnownIntrinsic entries are consistent");
1572 WidenedId = Widened->ID;
1573 WidenedFnTy = Widened->MakeFnTy(Context);
1574 } else {
1575 // This is not a known library function or intrinsic.
1576 return nullptr;
1577 }
1578
1579 // Check that the widened intrinsic is valid.
1580 SmallVector<Type *, 4> OverloadTys;
1581 [[maybe_unused]] bool IsValid =
1582 Intrinsic::isSignatureValid(ID: WidenedId, FT: WidenedFnTy, OverloadTys);
1583 assert(IsValid && "invalid widened intrinsic");
1584 // For known intrinsic functions, we create a second call to the same
1585 // intrinsic with a different type.
1586 SmallVector<Value *, 4> Args;
1587 // The last operand is the intrinsic itself, skip it.
1588 for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {
1589 Value *Arg = Call.getOperand(i_nocapture: I);
1590 Type *OrigArgTy = Arg->getType();
1591 Type *IntrinsicArgTy = WidenedFnTy->getParamType(i: I);
1592 if (OrigArgTy == IntrinsicArgTy) {
1593 Args.push_back(Elt: Arg); // The arg is passed as is.
1594 continue;
1595 }
1596 Type *ShadowArgTy = Config.getExtendedFPType(FT: Arg->getType());
1597 assert(ShadowArgTy &&
1598 "don't know how to get the shadow value for a non-FT");
1599 Value *Shadow = Map.getShadow(V: Arg);
1600 if (ShadowArgTy == IntrinsicArgTy) {
1601 // The shadow is the right type for the intrinsic.
1602 assert(Shadow->getType() == ShadowArgTy);
1603 Args.push_back(Elt: Shadow);
1604 continue;
1605 }
1606 // There is no intrinsic with his level of precision, truncate the shadow.
1607 Args.push_back(Elt: Builder.CreateFPTrunc(V: Shadow, DestTy: IntrinsicArgTy));
1608 }
1609 Value *IntrinsicCall = Builder.CreateIntrinsic(ID: WidenedId, OverloadTypes: OverloadTys, Args);
1610 return WidenedFnTy->getReturnType() == ExtendedVT
1611 ? IntrinsicCall
1612 : Builder.CreateFPExt(V: IntrinsicCall, DestTy: ExtendedVT);
1613}
1614
1615// Handle a CallBase, i.e. a function call, an inline asm sequence, or an
1616// invoke.
1617Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,
1618 Type *ExtendedVT,
1619 const TargetLibraryInfo &TLI,
1620 const ValueToShadowMap &Map,
1621 IRBuilder<> &Builder) {
1622 // We cannot look inside inline asm, just expand the result again.
1623 if (Call.isInlineAsm())
1624 return Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT);
1625
1626 // Intrinsics and library functions (e.g. sin, exp) are handled
1627 // specifically, because we know their semantics and can do better than
1628 // blindly calling them (e.g. compute the sinus in the actual shadow domain).
1629 if (Value *V =
1630 maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))
1631 return V;
1632
1633 // If the return tag matches that of the called function, read the extended
1634 // return value from the shadow ret ptr. Else, just extend the return value.
1635 Value *L =
1636 Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowRetTag, /*isVolatile=*/false);
1637 Value *HasShadowRet = Builder.CreateICmpEQ(
1638 LHS: L, RHS: Builder.CreatePtrToInt(V: Call.getCalledOperand(), DestTy: IntptrTy));
1639
1640 Value *ShadowRetVal = Builder.CreateLoad(
1641 Ty: ExtendedVT,
1642 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0),
1643 /*isVolatile=*/false);
1644 Value *Shadow = Builder.CreateSelect(C: HasShadowRet, True: ShadowRetVal,
1645 False: Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT));
1646 ++NumInstrumentedFTCalls;
1647 return Shadow;
1648}
1649
1650// Creates a shadow value for the given FT value. At that point all operands are
1651// guaranteed to be available.
1652Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(
1653 Instruction &Inst, const TargetLibraryInfo &TLI,
1654 const ValueToShadowMap &Map) {
1655 Type *VT = Inst.getType();
1656 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1657 assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");
1658
1659 if (auto *Load = dyn_cast<LoadInst>(Val: &Inst))
1660 return handleLoad(Load&: *Load, VT, ExtendedVT);
1661
1662 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
1663 // Insert after the call.
1664 BasicBlock::iterator It(Inst);
1665 IRBuilder<> Builder(Call->getParent(), ++It);
1666 Builder.SetCurrentDebugLocation(Call->getDebugLoc());
1667 return handleCallBase(Call&: *Call, VT, ExtendedVT, TLI, Map, Builder);
1668 }
1669
1670 if (auto *Invoke = dyn_cast<InvokeInst>(Val: &Inst)) {
1671 // The Invoke terminates the basic block, create a new basic block in
1672 // between the successful invoke and the next block.
1673 BasicBlock *InvokeBB = Invoke->getParent();
1674 BasicBlock *NextBB = Invoke->getNormalDest();
1675 BasicBlock *NewBB =
1676 BasicBlock::Create(Context, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
1677 Inst.replaceSuccessorWith(OldBB: NextBB, NewBB);
1678
1679 IRBuilder<> Builder(NewBB);
1680 Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());
1681 Value *Shadow = handleCallBase(Call&: *Invoke, VT, ExtendedVT, TLI, Map, Builder);
1682 Builder.CreateBr(Dest: NextBB);
1683 NewBB->replaceSuccessorsPhiUsesWith(Old: InvokeBB, New: NewBB);
1684 return Shadow;
1685 }
1686
1687 IRBuilder<> Builder(Inst.getNextNode());
1688 Builder.SetCurrentDebugLocation(Inst.getDebugLoc());
1689
1690 if (auto *Trunc = dyn_cast<FPTruncInst>(Val: &Inst))
1691 return handleTrunc(Trunc: *Trunc, VT, ExtendedVT, Map, Builder);
1692 if (auto *Ext = dyn_cast<FPExtInst>(Val: &Inst))
1693 return handleExt(Ext: *Ext, VT, ExtendedVT, Map, Builder);
1694
1695 if (auto *UnaryOp = dyn_cast<UnaryOperator>(Val: &Inst))
1696 return Builder.CreateUnOp(Opc: UnaryOp->getOpcode(),
1697 V: Map.getShadow(V: UnaryOp->getOperand(i_nocapture: 0)));
1698
1699 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: &Inst))
1700 return Builder.CreateBinOp(Opc: BinOp->getOpcode(),
1701 LHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 0)),
1702 RHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 1)));
1703
1704 if (isa<UIToFPInst>(Val: &Inst) || isa<SIToFPInst>(Val: &Inst)) {
1705 auto *Cast = cast<CastInst>(Val: &Inst);
1706 return Builder.CreateCast(Op: Cast->getOpcode(), V: Cast->getOperand(i_nocapture: 0),
1707 DestTy: ExtendedVT);
1708 }
1709
1710 if (auto *S = dyn_cast<SelectInst>(Val: &Inst))
1711 return Builder.CreateSelect(C: S->getCondition(),
1712 True: Map.getShadow(V: S->getTrueValue()),
1713 False: Map.getShadow(V: S->getFalseValue()));
1714
1715 if (auto *Freeze = dyn_cast<FreezeInst>(Val: &Inst))
1716 return Builder.CreateFreeze(V: Map.getShadow(V: Freeze->getOperand(i_nocapture: 0)));
1717
1718 if (auto *Extract = dyn_cast<ExtractElementInst>(Val: &Inst))
1719 return Builder.CreateExtractElement(
1720 Vec: Map.getShadow(V: Extract->getVectorOperand()), Idx: Extract->getIndexOperand());
1721
1722 if (auto *Insert = dyn_cast<InsertElementInst>(Val: &Inst))
1723 return Builder.CreateInsertElement(Vec: Map.getShadow(V: Insert->getOperand(i_nocapture: 0)),
1724 NewElt: Map.getShadow(V: Insert->getOperand(i_nocapture: 1)),
1725 Idx: Insert->getOperand(i_nocapture: 2));
1726
1727 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: &Inst))
1728 return Builder.CreateShuffleVector(V1: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 0)),
1729 V2: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 1)),
1730 Mask: Shuffle->getShuffleMask());
1731 // TODO: We could make aggregate object first class citizens. For now we
1732 // just extend the extracted value.
1733 if (auto *Extract = dyn_cast<ExtractValueInst>(Val: &Inst))
1734 return Builder.CreateFPExt(V: Extract, DestTy: ExtendedVT);
1735
1736 if (auto *BC = dyn_cast<BitCastInst>(Val: &Inst))
1737 return Builder.CreateFPExt(V: BC, DestTy: ExtendedVT);
1738
1739 report_fatal_error(reason: "Unimplemented support for " +
1740 Twine(Inst.getOpcodeName()));
1741}
1742
1743// Creates a shadow value for an instruction that defines a value of FT type.
1744// FT operands that do not already have shadow values are created recursively.
1745// The DFS is guaranteed to not loop as phis and arguments already have
1746// shadows.
1747void NumericalStabilitySanitizer::maybeCreateShadowValue(
1748 Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
1749 Type *VT = Root.getType();
1750 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1751 if (ExtendedVT == nullptr)
1752 return; // Not an FT value.
1753
1754 if (Map.hasShadow(V: &Root))
1755 return; // Shadow already exists.
1756
1757 assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");
1758
1759 std::vector<Instruction *> DfsStack(1, &Root);
1760 while (!DfsStack.empty()) {
1761 // Ensure that all operands to the instruction have shadows before
1762 // proceeding.
1763 Instruction *I = DfsStack.back();
1764 // The shadow for the instruction might have been created deeper in the DFS,
1765 // see `forward_use_with_two_uses` test.
1766 if (Map.hasShadow(V: I)) {
1767 DfsStack.pop_back();
1768 continue;
1769 }
1770
1771 bool MissingShadow = false;
1772 for (Value *Op : I->operands()) {
1773 Type *VT = Op->getType();
1774 if (!Config.getExtendedFPType(FT: VT))
1775 continue; // Not an FT value.
1776 if (Map.hasShadow(V: Op))
1777 continue; // Shadow is already available.
1778 MissingShadow = true;
1779 DfsStack.push_back(x: cast<Instruction>(Val: Op));
1780 }
1781 if (MissingShadow)
1782 continue; // Process operands and come back to this instruction later.
1783
1784 // All operands have shadows. Create a shadow for the current value.
1785 Value *Shadow = createShadowValueWithOperandsAvailable(Inst&: *I, TLI, Map);
1786 Map.setShadow(V&: *I, Shadow&: *Shadow);
1787 DfsStack.pop_back();
1788 }
1789}
1790
1791// A floating-point store needs its value and type written to shadow memory.
1792void NumericalStabilitySanitizer::propagateFTStore(
1793 StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {
1794 Value *StoredValue = Store.getValueOperand();
1795 IRBuilder<> Builder(&Store);
1796 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1797 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1798 Value *ShadowPtr = Builder.CreateCall(
1799 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1800 Args: {Store.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1801
1802 Value *StoredShadow = Map.getShadow(V: StoredValue);
1803 if (!Store.getParent()->getParent()->hasOptNone()) {
1804 // Only check stores when optimizing, because non-optimized code generates
1805 // too many stores to the stack, creating false positives.
1806 if (ClCheckStores) {
1807 StoredShadow = emitCheck(V: StoredValue, ShadowV: StoredShadow, Builder,
1808 Loc: CheckLoc::makeStore(Address: Store.getPointerOperand()));
1809 ++NumInstrumentedFTStores;
1810 }
1811 }
1812
1813 Builder.CreateAlignedStore(Val: StoredShadow, Ptr: ShadowPtr, Align: Align(1),
1814 isVolatile: Store.isVolatile());
1815}
1816
1817// A non-ft store needs to invalidate shadow memory. Exceptions are:
1818// - memory transfers of floating-point data through other pointer types (llvm
1819// optimization passes transform `*(float*)a = *(float*)b` into
1820// `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.
1821// - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted
1822// ints. Note that this is not really necessary because if the value is
1823// unknown the framework will re-extend it on load anyway. It just felt
1824// easier to debug tests with vectors of FTs.
1825void NumericalStabilitySanitizer::propagateNonFTStore(
1826 StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {
1827 Value *PtrOp = Store.getPointerOperand();
1828 IRBuilder<> Builder(Store.getNextNode());
1829 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1830 Value *Dst = PtrOp;
1831 TypeSize SlotSize = DL.getTypeStoreSize(Ty: VT);
1832 assert(!SlotSize.isScalable() && "unsupported");
1833 const auto LoadSizeBytes = SlotSize.getFixedValue();
1834 Value *ValueSize = Constant::getIntegerValue(
1835 Ty: IntptrTy, V: APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));
1836
1837 ++NumInstrumentedNonFTStores;
1838 Value *StoredValue = Store.getValueOperand();
1839 if (LoadInst *Load = dyn_cast<LoadInst>(Val: StoredValue)) {
1840 // TODO: Handle the case when the value is from a phi.
1841 // This is a memory transfer with memcpy semantics. Copy the type and
1842 // value from the source. Note that we cannot use __nsan_copy_values()
1843 // here, because that will not work when there is a write to memory in
1844 // between the load and the store, e.g. in the case of a swap.
1845 Type *ShadowTypeIntTy = Type::getIntNTy(C&: Context, N: 8 * LoadSizeBytes);
1846 Type *ShadowValueIntTy =
1847 Type::getIntNTy(C&: Context, N: 8 * kShadowScale * LoadSizeBytes);
1848 IRBuilder<> LoadBuilder(Load->getNextNode());
1849 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1850 Value *LoadSrc = Load->getPointerOperand();
1851 // Read the shadow type and value at load time. The type has the same size
1852 // as the FT value, the value has twice its size.
1853 // TODO: cache them to avoid re-creating them when a load is used by
1854 // several stores. Maybe create them like the FT shadows when a load is
1855 // encountered.
1856 Value *RawShadowType = LoadBuilder.CreateAlignedLoad(
1857 Ty: ShadowTypeIntTy,
1858 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {LoadSrc}), Align: Align(1),
1859 /*isVolatile=*/false);
1860 Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(
1861 Ty: ShadowValueIntTy,
1862 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {LoadSrc}), Align: Align(1),
1863 /*isVolatile=*/false);
1864
1865 // Write back the shadow type and value at store time.
1866 Builder.CreateAlignedStore(
1867 Val: RawShadowType, Ptr: Builder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {Dst}),
1868 Align: Align(1),
1869 /*isVolatile=*/false);
1870 Builder.CreateAlignedStore(Val: RawShadowValue,
1871 Ptr: Builder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {Dst}),
1872 Align: Align(1),
1873 /*isVolatile=*/false);
1874
1875 ++NumInstrumentedNonFTMemcpyStores;
1876 return;
1877 }
1878 // ClPropagateNonFTConstStoresAsFT is by default false.
1879 if (Constant *C; ClPropagateNonFTConstStoresAsFT &&
1880 (C = dyn_cast<Constant>(Val: StoredValue))) {
1881 // This might be a fp constant stored as an int. Bitcast and store if it has
1882 // appropriate size.
1883 Type *BitcastTy = nullptr; // The FT type to bitcast to.
1884 if (isa<ConstantInt, ConstantDataVector>(Val: C)) {
1885 switch (C->getType()->getScalarSizeInBits()) {
1886 case 32:
1887 BitcastTy = Type::getFloatTy(C&: Context);
1888 break;
1889 case 64:
1890 BitcastTy = Type::getDoubleTy(C&: Context);
1891 break;
1892 case 80:
1893 BitcastTy = Type::getX86_FP80Ty(C&: Context);
1894 break;
1895 default:
1896 break;
1897 }
1898
1899 if (BitcastTy)
1900 if (auto *VectorTy = dyn_cast<VectorType>(Val: C->getType()))
1901 BitcastTy = VectorType::get(ElementType: BitcastTy, EC: VectorTy->getElementCount());
1902 }
1903 if (BitcastTy) {
1904 const MemoryExtents Extents = getMemoryExtentsOrDie(FT: BitcastTy);
1905 Value *ShadowPtr = Builder.CreateCall(
1906 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1907 Args: {PtrOp, ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1908 // Bitcast the integer value to the appropriate FT type and extend to 2FT.
1909 Type *ExtVT = Config.getExtendedFPType(FT: BitcastTy);
1910 Value *Shadow =
1911 Builder.CreateFPExt(V: Builder.CreateBitCast(V: C, DestTy: BitcastTy), DestTy: ExtVT);
1912 Builder.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1),
1913 isVolatile: Store.isVolatile());
1914 return;
1915 }
1916 }
1917 // All other stores just reset the shadow value to unknown.
1918 Builder.CreateCall(Callee: NsanSetUnknownFns.getFallback(), Args: {Dst, ValueSize});
1919}
1920
1921void NumericalStabilitySanitizer::propagateShadowValues(
1922 Instruction &Inst, const TargetLibraryInfo &TLI,
1923 const ValueToShadowMap &Map) {
1924 if (auto *Store = dyn_cast<StoreInst>(Val: &Inst)) {
1925 Value *StoredValue = Store->getValueOperand();
1926 Type *VT = StoredValue->getType();
1927 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1928 if (ExtendedVT == nullptr)
1929 return propagateNonFTStore(Store&: *Store, VT, Map);
1930 return propagateFTStore(Store&: *Store, VT, ExtendedVT, Map);
1931 }
1932
1933 if (auto *FCmp = dyn_cast<FCmpInst>(Val: &Inst)) {
1934 emitFCmpCheck(FCmp&: *FCmp, Map);
1935 return;
1936 }
1937
1938 if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
1939 maybeAddSuffixForNsanInterface(CI: CB);
1940 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
1941 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
1942 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
1943 instrumentMemIntrinsic(MI);
1944 return;
1945 }
1946 populateShadowStack(CI&: *CB, TLI, Map);
1947 return;
1948 }
1949
1950 if (auto *RetInst = dyn_cast<ReturnInst>(Val: &Inst)) {
1951 if (!ClCheckRet)
1952 return;
1953
1954 Value *RV = RetInst->getReturnValue();
1955 if (RV == nullptr)
1956 return; // This is a `ret void`.
1957 Type *VT = RV->getType();
1958 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1959 if (ExtendedVT == nullptr)
1960 return; // Not an FT ret.
1961 Value *RVShadow = Map.getShadow(V: RV);
1962 IRBuilder<> Builder(RetInst);
1963
1964 RVShadow = emitCheck(V: RV, ShadowV: RVShadow, Builder, Loc: CheckLoc::makeRet());
1965 ++NumInstrumentedFTRets;
1966 // Store tag.
1967 Value *FnAddr =
1968 Builder.CreatePtrToInt(V: Inst.getParent()->getParent(), DestTy: IntptrTy);
1969 Builder.CreateStore(Val: FnAddr, Ptr: NsanShadowRetTag);
1970 // Store value.
1971 Value *ShadowRetValPtr =
1972 Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0);
1973 Builder.CreateStore(Val: RVShadow, Ptr: ShadowRetValPtr);
1974 return;
1975 }
1976
1977 if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(Val: &Inst)) {
1978 Value *V = Insert->getOperand(i_nocapture: 1);
1979 Type *VT = V->getType();
1980 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1981 if (ExtendedVT == nullptr)
1982 return;
1983 IRBuilder<> Builder(Insert);
1984 emitCheck(V, ShadowV: Map.getShadow(V), Builder, Loc: CheckLoc::makeInsert());
1985 return;
1986 }
1987}
1988
1989// Moves fast math flags from the function to individual instructions, and
1990// removes the attribute from the function.
1991// TODO: Make this controllable with a flag.
1992static void moveFastMathFlags(Function &F,
1993 std::vector<Instruction *> &Instructions) {
1994 FastMathFlags FMF;
1995#define MOVE_FLAG(attr, setter) \
1996 if (F.getFnAttribute(attr).getValueAsString() == "true") { \
1997 F.removeFnAttr(attr); \
1998 FMF.set##setter(); \
1999 }
2000 MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)
2001#undef MOVE_FLAG
2002
2003 for (Instruction *I : Instructions)
2004 if (isa<FPMathOperator>(Val: I))
2005 I->setFastMathFlags(FMF);
2006}
2007
2008bool NumericalStabilitySanitizer::sanitizeFunction(
2009 Function &F, const TargetLibraryInfo &TLI) {
2010 if (!F.hasFnAttribute(Kind: Attribute::SanitizeNumericalStability) ||
2011 F.isDeclaration())
2012 return false;
2013
2014 // This is required to prevent instrumenting call to __nsan_init from within
2015 // the module constructor.
2016 if (F.getName() == kNsanModuleCtorName)
2017 return false;
2018
2019 // The instrumentation maintains:
2020 // - for each IR value `v` of floating-point (or vector floating-point) type
2021 // FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.
2022 // double for float and f128 for double).
2023 // - A shadow memory, which stores `s(v)` for any `v` that has been stored,
2024 // along with a shadow memory tag, which stores whether the value in the
2025 // corresponding shadow memory is valid. Note that this might be
2026 // incorrect if a non-instrumented function stores to memory, or if
2027 // memory is stored to through a char pointer.
2028 // - A shadow stack, which holds `s(v)` for any floating-point argument `v`
2029 // of a call to an instrumented function. This allows
2030 // instrumented functions to retrieve the shadow values for their
2031 // arguments.
2032 // Because instrumented functions can be called from non-instrumented
2033 // functions, the stack needs to include a tag so that the instrumented
2034 // function knows whether shadow values are available for their
2035 // parameters (i.e. whether is was called by an instrumented function).
2036 // When shadow arguments are not available, they have to be recreated by
2037 // extending the precision of the non-shadow arguments to the non-shadow
2038 // value. Non-instrumented functions do not modify (or even know about) the
2039 // shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow
2040 // stack tag is __nsan_shadow_args_tag. The tag is any unique identifier
2041 // for the function (we use the address of the function). Both variables
2042 // are thread local.
2043 // Example:
2044 // calls shadow stack tag shadow stack
2045 // =======================================================================
2046 // non_instrumented_1() 0 0
2047 // |
2048 // v
2049 // instrumented_2(float a) 0 0
2050 // |
2051 // v
2052 // instrumented_3(float b, double c) &instrumented_3 s(b),s(c)
2053 // |
2054 // v
2055 // instrumented_4(float d) &instrumented_4 s(d)
2056 // |
2057 // v
2058 // non_instrumented_5(float e) &non_instrumented_5 s(e)
2059 // |
2060 // v
2061 // instrumented_6(float f) &non_instrumented_5 s(e)
2062 //
2063 // On entry, instrumented_2 checks whether the tag corresponds to its
2064 // function ptr.
2065 // Note that functions reset the tag to 0 after reading shadow parameters.
2066 // This ensures that the function does not erroneously read invalid data if
2067 // called twice in the same stack, once from an instrumented function and
2068 // once from an uninstrumented one. For example, in the following example,
2069 // resetting the tag in (A) ensures that (B) does not reuse the same the
2070 // shadow arguments (which would be incorrect).
2071 // instrumented_1(float a)
2072 // |
2073 // v
2074 // instrumented_2(float b) (A)
2075 // |
2076 // v
2077 // non_instrumented_3()
2078 // |
2079 // v
2080 // instrumented_2(float b) (B)
2081 //
2082 // - A shadow return slot. Any function that returns a floating-point value
2083 // places a shadow return value in __nsan_shadow_ret_val. Again, because
2084 // we might be calling non-instrumented functions, this value is guarded
2085 // by __nsan_shadow_ret_tag marker indicating which instrumented function
2086 // placed the value in __nsan_shadow_ret_val, so that the caller can check
2087 // that this corresponds to the callee. Both variables are thread local.
2088 //
2089 // For example, in the following example, the instrumentation in
2090 // `instrumented_1` rejects the shadow return value from `instrumented_3`
2091 // because it is not tagged as expected (`&instrumented_3` instead of
2092 // `non_instrumented_2`):
2093 //
2094 // instrumented_1()
2095 // |
2096 // v
2097 // float non_instrumented_2()
2098 // |
2099 // v
2100 // float instrumented_3()
2101 //
2102 // Calls of known math functions (sin, cos, exp, ...) are duplicated to call
2103 // their overload on the shadow type.
2104
2105 // Collect all instructions before processing, as creating shadow values
2106 // creates new instructions inside the function.
2107 std::vector<Instruction *> OriginalInstructions;
2108 for (BasicBlock &BB : F)
2109 for (Instruction &Inst : BB)
2110 OriginalInstructions.emplace_back(args: &Inst);
2111
2112 moveFastMathFlags(F, Instructions&: OriginalInstructions);
2113 ValueToShadowMap ValueToShadow(Config);
2114
2115 // In the first pass, we create shadow values for all FT function arguments
2116 // and all phis. This ensures that the DFS of the next pass does not have
2117 // any loops.
2118 std::vector<PHINode *> OriginalPhis;
2119 createShadowArguments(F, TLI, Map&: ValueToShadow);
2120 for (Instruction *I : OriginalInstructions) {
2121 if (PHINode *Phi = dyn_cast<PHINode>(Val: I)) {
2122 if (PHINode *Shadow = maybeCreateShadowPhi(Phi&: *Phi, TLI)) {
2123 OriginalPhis.push_back(x: Phi);
2124 ValueToShadow.setShadow(V&: *Phi, Shadow&: *Shadow);
2125 }
2126 }
2127 }
2128
2129 // Create shadow values for all instructions creating FT values.
2130 for (Instruction *I : OriginalInstructions)
2131 maybeCreateShadowValue(Root&: *I, TLI, Map&: ValueToShadow);
2132
2133 // Propagate shadow values across stores, calls and rets.
2134 for (Instruction *I : OriginalInstructions)
2135 propagateShadowValues(Inst&: *I, TLI, Map: ValueToShadow);
2136
2137 // The last pass populates shadow phis with shadow values.
2138 for (PHINode *Phi : OriginalPhis) {
2139 PHINode *ShadowPhi = cast<PHINode>(Val: ValueToShadow.getShadow(V: Phi));
2140 for (unsigned I : seq(Size: Phi->getNumOperands())) {
2141 Value *V = Phi->getOperand(i_nocapture: I);
2142 Value *Shadow = ValueToShadow.getShadow(V);
2143 BasicBlock *IncomingBB = Phi->getIncomingBlock(i: I);
2144 // For some instructions (e.g. invoke), we create the shadow in a separate
2145 // block, different from the block where the original value is created.
2146 // In that case, the shadow phi might need to refer to this block instead
2147 // of the original block.
2148 // Note that this can only happen for instructions as constant shadows are
2149 // always created in the same block.
2150 ShadowPhi->addIncoming(V: Shadow, BB: IncomingBB);
2151 }
2152 }
2153
2154 return !ValueToShadow.empty();
2155}
2156
2157static uint64_t GetMemOpSize(Value *V) {
2158 uint64_t OpSize = 0;
2159 if (Constant *C = dyn_cast<Constant>(Val: V)) {
2160 auto *CInt = dyn_cast<ConstantInt>(Val: C);
2161 if (CInt && CInt->getValue().getBitWidth() <= 64)
2162 OpSize = CInt->getValue().getZExtValue();
2163 }
2164
2165 return OpSize;
2166}
2167
2168// Instrument the memory intrinsics so that they properly modify the shadow
2169// memory.
2170bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
2171 IRBuilder<> Builder(MI);
2172 if (auto *M = dyn_cast<MemSetInst>(Val: MI)) {
2173 FunctionCallee SetUnknownFn =
2174 NsanSetUnknownFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2175 if (SetUnknownFn.getFunctionType()->getNumParams() == 1)
2176 Builder.CreateCall(Callee: SetUnknownFn, Args: {/*Address=*/M->getArgOperand(i: 0)});
2177 else
2178 Builder.CreateCall(Callee: SetUnknownFn,
2179 Args: {/*Address=*/M->getArgOperand(i: 0),
2180 /*Size=*/Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2181 DestTy: IntptrTy, isSigned: false)});
2182
2183 } else if (auto *M = dyn_cast<MemTransferInst>(Val: MI)) {
2184 FunctionCallee CopyFn =
2185 NsanCopyFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2186
2187 if (CopyFn.getFunctionType()->getNumParams() == 2)
2188 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2189 /*Source=*/M->getArgOperand(i: 1)});
2190 else
2191 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2192 /*Source=*/M->getArgOperand(i: 1),
2193 /*Size=*/
2194 Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2195 DestTy: IntptrTy, isSigned: false)});
2196 }
2197 return false;
2198}
2199
2200void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {
2201 Function *Fn = CI->getCalledFunction();
2202 if (Fn == nullptr)
2203 return;
2204
2205 if (!Fn->getName().starts_with(Prefix: "__nsan_"))
2206 return;
2207
2208 if (Fn->getName() == "__nsan_dump_shadow_mem") {
2209 assert(CI->arg_size() == 4 &&
2210 "invalid prototype for __nsan_dump_shadow_mem");
2211 // __nsan_dump_shadow_mem requires an extra parameter with the dynamic
2212 // configuration:
2213 // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)
2214 // | shadow_type_id_for_double
2215 const uint64_t shadow_value_type_ids =
2216 (static_cast<size_t>(Config.byValueType(VT: kLongDouble).getNsanTypeId())
2217 << 16) |
2218 (static_cast<size_t>(Config.byValueType(VT: kDouble).getNsanTypeId())
2219 << 8) |
2220 static_cast<size_t>(Config.byValueType(VT: kFloat).getNsanTypeId());
2221 CI->setArgOperand(i: 3, v: ConstantInt::get(Ty: IntptrTy, V: shadow_value_type_ids));
2222 }
2223}
2224