1//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the instrumentation pass for the numerical sanitizer.
10// Conceptually the pass injects shadow computations using higher precision
11// types and inserts consistency checks. For details see the paper
12// https://arxiv.org/abs/2102.12782.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
17#include "InstrumentationOptions.h"
18
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/IntrinsicInst.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/Type.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Regex.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Transforms/Utils/BasicBlockUtils.h"
39#include "llvm/Transforms/Utils/Instrumentation.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43#include <cstdint>
44
45using namespace llvm;
46
47#define DEBUG_TYPE "nsan"
48
49STATISTIC(NumInstrumentedFTLoads,
50 "Number of instrumented floating-point loads");
51
52STATISTIC(NumInstrumentedFTCalls,
53 "Number of instrumented floating-point calls");
54STATISTIC(NumInstrumentedFTRets,
55 "Number of instrumented floating-point returns");
56STATISTIC(NumInstrumentedFTStores,
57 "Number of instrumented floating-point stores");
58STATISTIC(NumInstrumentedNonFTStores,
59 "Number of instrumented non floating-point stores");
60STATISTIC(
61 NumInstrumentedNonFTMemcpyStores,
62 "Number of instrumented non floating-point stores with memcpy semantics");
63STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
64
65constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
66constexpr StringLiteral kNsanInitName("__nsan_init");
67
68// The following values must be kept in sync with the runtime.
69constexpr int kShadowScale = 2;
70constexpr int kMaxVectorWidth = 8;
71constexpr int kMaxNumArgs = 128;
72constexpr int kMaxShadowTypeSizeBytes = 16; // fp128
73
74namespace {
75
76// Defines the characteristics (type id, type, and floating-point semantics)
77// attached for all possible shadow types.
78class ShadowTypeConfig {
79public:
80 static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
81
82 // The LLVM Type corresponding to the shadow type.
83 virtual Type *getType(LLVMContext &Context) const = 0;
84
85 // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
86 virtual char getNsanTypeId() const = 0;
87
88 virtual ~ShadowTypeConfig() = default;
89};
90
91template <char NsanTypeId>
92class ShadowTypeConfigImpl : public ShadowTypeConfig {
93public:
94 char getNsanTypeId() const override { return NsanTypeId; }
95 static constexpr char kNsanTypeId = NsanTypeId;
96};
97
98// `double` (`d`) shadow type.
99class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
100 Type *getType(LLVMContext &Context) const override {
101 return Type::getDoubleTy(C&: Context);
102 }
103};
104
105// `x86_fp80` (`l`) shadow type: X86 long double.
106class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
107 Type *getType(LLVMContext &Context) const override {
108 return Type::getX86_FP80Ty(C&: Context);
109 }
110};
111
112// `fp128` (`q`) shadow type.
113class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
114 Type *getType(LLVMContext &Context) const override {
115 return Type::getFP128Ty(C&: Context);
116 }
117};
118
119// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
120class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
121 Type *getType(LLVMContext &Context) const override {
122 return Type::getPPC_FP128Ty(C&: Context);
123 }
124};
125
126// Creates a ShadowTypeConfig given its type id.
127std::unique_ptr<ShadowTypeConfig>
128ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
129 switch (TypeId) {
130 case F64ShadowConfig::kNsanTypeId:
131 return std::make_unique<F64ShadowConfig>();
132 case F80ShadowConfig::kNsanTypeId:
133 return std::make_unique<F80ShadowConfig>();
134 case F128ShadowConfig::kNsanTypeId:
135 return std::make_unique<F128ShadowConfig>();
136 case PPC128ShadowConfig::kNsanTypeId:
137 return std::make_unique<PPC128ShadowConfig>();
138 }
139 report_fatal_error(reason: "nsan: invalid shadow type id '" + Twine(TypeId) + "'");
140}
141
142// An enum corresponding to shadow value types. Used as indices in arrays, so
143// not an `enum class`.
144enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
145
146// If `FT` corresponds to a primitive FTValueType, return it.
147static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
148 if (FT->isFloatTy())
149 return kFloat;
150 if (FT->isDoubleTy())
151 return kDouble;
152 if (FT->isX86_FP80Ty())
153 return kLongDouble;
154 return {};
155}
156
157// Returns the LLVM type for an FTValueType.
158static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
159 switch (VT) {
160 case kFloat:
161 return Type::getFloatTy(C&: Context);
162 case kDouble:
163 return Type::getDoubleTy(C&: Context);
164 case kLongDouble:
165 return Type::getX86_FP80Ty(C&: Context);
166 case kNumValueTypes:
167 return nullptr;
168 }
169 llvm_unreachable("Unhandled FTValueType enum");
170}
171
172// Returns the type name for an FTValueType.
173static const char *typeNameFromFTValueType(FTValueType VT) {
174 switch (VT) {
175 case kFloat:
176 return "float";
177 case kDouble:
178 return "double";
179 case kLongDouble:
180 return "longdouble";
181 case kNumValueTypes:
182 return nullptr;
183 }
184 llvm_unreachable("Unhandled FTValueType enum");
185}
186
187// A specific mapping configuration of application type to shadow type for nsan
188// (see -nsan-shadow-mapping flag).
189class MappingConfig {
190public:
191 MappingConfig(const InstrumentationOptions &Opts, LLVMContext &C)
192 : Context(C) {
193 if (Opts.nsan_shadow_type_mapping.size() != 3)
194 report_fatal_error(reason: "Invalid nsan mapping: " +
195 Twine(Opts.nsan_shadow_type_mapping));
196 unsigned ShadowTypeSizeBits[kNumValueTypes];
197 for (int VT = 0; VT < kNumValueTypes; ++VT) {
198 auto Config =
199 ShadowTypeConfig::fromNsanTypeId(TypeId: Opts.nsan_shadow_type_mapping[VT]);
200 if (!Config)
201 report_fatal_error(reason: "Failed to get ShadowTypeConfig for " +
202 Twine(Opts.nsan_shadow_type_mapping[VT]));
203 const unsigned AppTypeSize =
204 typeFromFTValueType(VT: static_cast<FTValueType>(VT), Context)
205 ->getScalarSizeInBits();
206 const unsigned ShadowTypeSize =
207 Config->getType(Context)->getScalarSizeInBits();
208 // Check that the shadow type size is at most kShadowScale times the
209 // application type size, so that shadow memory compoutations are valid.
210 if (ShadowTypeSize > kShadowScale * AppTypeSize)
211 report_fatal_error(reason: "Invalid nsan mapping f" + Twine(AppTypeSize) +
212 "->f" + Twine(ShadowTypeSize) +
213 ": The shadow type size should be at most " +
214 Twine(kShadowScale) +
215 " times the application type size");
216 ShadowTypeSizeBits[VT] = ShadowTypeSize;
217 Configs[VT] = std::move(Config);
218 }
219
220 // Check that the mapping is monotonous. This is required because if one
221 // does an fpextend of `float->long double` in application code, nsan is
222 // going to do an fpextend of `shadow(float) -> shadow(long double)` in
223 // shadow code. This will fail in `qql` mode, since nsan would be
224 // fpextending `f128->long`, which is invalid.
225 // TODO: Relax this.
226 if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
227 ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
228 report_fatal_error(reason: "Invalid nsan mapping: { float->f" +
229 Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
230 Twine(ShadowTypeSizeBits[kDouble]) +
231 "; long double->f" +
232 Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
233 }
234
235 const ShadowTypeConfig &byValueType(FTValueType VT) const {
236 assert(VT < FTValueType::kNumValueTypes && "invalid value type");
237 return *Configs[VT];
238 }
239
240 // Returns the extended shadow type for a given application type.
241 Type *getExtendedFPType(Type *FT) const {
242 if (const auto VT = ftValueTypeFromType(FT))
243 return Configs[*VT]->getType(Context);
244 if (FT->isVectorTy()) {
245 auto *VecTy = cast<VectorType>(Val: FT);
246 // TODO: add support for scalable vector types.
247 if (VecTy->isScalableTy())
248 return nullptr;
249 Type *ExtendedScalar = getExtendedFPType(FT: VecTy->getElementType());
250 return ExtendedScalar
251 ? VectorType::get(ElementType: ExtendedScalar, EC: VecTy->getElementCount())
252 : nullptr;
253 }
254 return nullptr;
255 }
256
257private:
258 LLVMContext &Context;
259 std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
260};
261
262// The memory extents of a type specifies how many elements of a given
263// FTValueType needs to be stored when storing this type.
264struct MemoryExtents {
265 FTValueType ValueType;
266 uint64_t NumElts;
267};
268
269static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
270 if (const auto VT = ftValueTypeFromType(FT))
271 return {.ValueType: *VT, .NumElts: 1};
272 if (auto *VecTy = dyn_cast<VectorType>(Val: FT)) {
273 const auto ScalarExtents = getMemoryExtentsOrDie(FT: VecTy->getElementType());
274 return {.ValueType: ScalarExtents.ValueType,
275 .NumElts: ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
276 }
277 llvm_unreachable("invalid value type");
278}
279
280// The location of a check. Passed as parameters to runtime checking functions.
281class CheckLoc {
282public:
283 // Creates a location that references an application memory location.
284 static CheckLoc makeStore(Value *Address) {
285 CheckLoc Result(kStore);
286 Result.Address = Address;
287 return Result;
288 }
289 static CheckLoc makeLoad(Value *Address) {
290 CheckLoc Result(kLoad);
291 Result.Address = Address;
292 return Result;
293 }
294
295 // Creates a location that references an argument, given by id.
296 static CheckLoc makeArg(int ArgId) {
297 CheckLoc Result(kArg);
298 Result.ArgId = ArgId;
299 return Result;
300 }
301
302 // Creates a location that references the return value of a function.
303 static CheckLoc makeRet() { return CheckLoc(kRet); }
304
305 // Creates a location that references a vector insert.
306 static CheckLoc makeInsert() { return CheckLoc(kInsert); }
307
308 // Returns the CheckType of location this refers to, as an integer-typed LLVM
309 // IR value.
310 Value *getType(LLVMContext &C) const {
311 return ConstantInt::get(Ty: Type::getInt32Ty(C), V: static_cast<int>(CheckTy));
312 }
313
314 // Returns a CheckType-specific value representing details of the location
315 // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
316 // IR value.
317 Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
318 switch (CheckTy) {
319 case kUnknown:
320 llvm_unreachable("unknown type");
321 case kRet:
322 case kInsert:
323 return ConstantInt::get(Ty: IntptrTy, V: 0);
324 case kArg:
325 return ConstantInt::get(Ty: IntptrTy, V: ArgId);
326 case kLoad:
327 case kStore:
328 return Builder.CreatePtrToInt(V: Address, DestTy: IntptrTy);
329 }
330 llvm_unreachable("Unhandled CheckType enum");
331 }
332
333private:
334 // Must be kept in sync with the runtime,
335 // see compiler-rt/lib/nsan/nsan_stats.h
336 enum CheckType {
337 kUnknown = 0,
338 kRet,
339 kArg,
340 kLoad,
341 kStore,
342 kInsert,
343 };
344 explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
345
346 Value *Address = nullptr;
347 const CheckType CheckTy;
348 int ArgId = -1;
349};
350
351// A map of LLVM IR values to shadow LLVM IR values.
352class ValueToShadowMap {
353public:
354 explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}
355
356 ValueToShadowMap(const ValueToShadowMap &) = delete;
357 ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;
358
359 // Sets the shadow value for a value. Asserts that the value does not already
360 // have a value.
361 void setShadow(Value &V, Value &Shadow) {
362 [[maybe_unused]] const bool Inserted = Map.try_emplace(Key: &V, Args: &Shadow).second;
363 LLVM_DEBUG({
364 if (!Inserted) {
365 if (auto *I = dyn_cast<Instruction>(&V))
366 errs() << I->getFunction()->getName() << ": ";
367 errs() << "duplicate shadow (" << &V << "): ";
368 V.dump();
369 }
370 });
371 assert(Inserted && "duplicate shadow");
372 }
373
374 // Returns true if the value already has a shadow (including if the value is a
375 // constant). If true, calling getShadow() is valid.
376 bool hasShadow(Value *V) const { return isa<Constant>(Val: V) || Map.contains(Val: V); }
377
378 // Returns the shadow value for a given value. Asserts that the value has
379 // a shadow value. Lazily creates shadows for constant values.
380 Value *getShadow(Value *V) const {
381 if (Constant *C = dyn_cast<Constant>(Val: V))
382 return getShadowConstant(C);
383 return Map.find(Val: V)->second;
384 }
385
386 bool empty() const { return Map.empty(); }
387
388private:
389 // Extends a constant application value to its shadow counterpart.
390 APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
391 bool LosesInfo = false;
392 CV.convert(ToSemantics: To, RM: APFloatBase::rmTowardZero, losesInfo: &LosesInfo);
393 return CV;
394 }
395
396 // Returns the shadow constant for the given application constant.
397 Constant *getShadowConstant(Constant *C) const {
398 if (UndefValue *U = dyn_cast<UndefValue>(Val: C)) {
399 return UndefValue::get(T: Config.getExtendedFPType(FT: U->getType()));
400 }
401 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
402 // Floating-point constants.
403 Type *Ty = Config.getExtendedFPType(FT: CFP->getType());
404 return ConstantFP::get(
405 Ty, V: extendConstantFP(CV: CFP->getValueAPF(),
406 To: Ty->getScalarType()->getFltSemantics()));
407 }
408 // Vector, array, or aggregate constants.
409 if (C->getType()->isVectorTy()) {
410 SmallVector<Constant *, 8> Elements;
411 for (int I = 0, E = cast<VectorType>(Val: C->getType())
412 ->getElementCount()
413 .getFixedValue();
414 I < E; ++I)
415 Elements.push_back(Elt: getShadowConstant(C: C->getAggregateElement(Elt: I)));
416 return ConstantVector::get(V: Elements);
417 }
418 llvm_unreachable("unimplemented");
419 }
420
421 const MappingConfig &Config;
422 DenseMap<Value *, Value *> Map;
423};
424
425class NsanMemOpFn {
426public:
427 NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,
428 size_t NumArgs);
429 FunctionCallee getFunctionFor(uint64_t MemOpSize) const;
430 FunctionCallee getFallback() const;
431
432private:
433 SmallVector<FunctionCallee> Funcs;
434 size_t NumSizedFuncs;
435};
436
437NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,
438 StringRef Fallback, size_t NumArgs) {
439 LLVMContext &Ctx = M.getContext();
440 AttributeList Attr;
441 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
442 Type *PtrTy = PointerType::getUnqual(C&: Ctx);
443 Type *VoidTy = Type::getVoidTy(C&: Ctx);
444 IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx);
445 FunctionType *SizedFnTy = nullptr;
446
447 NumSizedFuncs = Sized.size();
448
449 // First entry is fallback function
450 if (NumArgs == 3) {
451 Funcs.push_back(
452 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy));
453 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy, PtrTy}, isVarArg: false);
454 } else if (NumArgs == 2) {
455 Funcs.push_back(
456 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: IntptrTy));
457 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false);
458 } else {
459 llvm_unreachable("Unexpected value of sized functions arguments");
460 }
461
462 for (size_t i = 0; i < NumSizedFuncs; ++i)
463 Funcs.push_back(Elt: M.getOrInsertFunction(Name: Sized[i], T: SizedFnTy, AttributeList: Attr));
464}
465
466FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {
467 // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the
468 // following code assumes that the number of functions in `Func` is sufficient
469 assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");
470
471 size_t Idx =
472 MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));
473
474 return Funcs[Idx];
475}
476
477FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }
478
479/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
480/// API function declarations into the module if they don't exist already.
481/// Instantiating ensures the __nsan_init function is in the list of global
482/// constructors for the module.
483class NumericalStabilitySanitizer {
484public:
485 NumericalStabilitySanitizer(const InstrumentationOptions &Opts, Module &M);
486 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
487
488private:
489 bool instrumentMemIntrinsic(MemIntrinsic *MI);
490 void maybeAddSuffixForNsanInterface(CallBase *CI);
491 bool addrPointsToConstantData(Value *Addr);
492 void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
493 ValueToShadowMap &Map);
494 Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
495 const TargetLibraryInfo &TLI,
496 const ValueToShadowMap &Map);
497 PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
498 void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
499 ValueToShadowMap &Map);
500
501 void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
502 const ValueToShadowMap &Map);
503
504 void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
505 const ValueToShadowMap &Map);
506 Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
507 CheckLoc Loc);
508 Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
509 CheckLoc Loc);
510 void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
511
512 // Value creation handlers.
513 Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
514 Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
515 const TargetLibraryInfo &TLI,
516 const ValueToShadowMap &Map, IRBuilder<> &Builder);
517 Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
518 const TargetLibraryInfo &TLI,
519 const ValueToShadowMap &Map,
520 IRBuilder<> &Builder);
521 Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
522 const ValueToShadowMap &Map, IRBuilder<> &Builder);
523 Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,
524 const ValueToShadowMap &Map, IRBuilder<> &Builder);
525
526 // Value propagation handlers.
527 void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
528 const ValueToShadowMap &Map);
529 void propagateNonFTStore(StoreInst &Store, Type *VT,
530 const ValueToShadowMap &Map);
531
532 const InstrumentationOptions &Opts;
533 const DataLayout &DL;
534 LLVMContext &Context;
535 MappingConfig Config;
536 IntegerType *IntptrTy = nullptr;
537
538 // TODO: Use std::array instead?
539 FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};
540 FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};
541 FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};
542 FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};
543
544 NsanMemOpFn NsanCopyFns;
545 NsanMemOpFn NsanSetUnknownFns;
546
547 FunctionCallee NsanGetRawShadowTypePtr;
548 FunctionCallee NsanGetRawShadowPtr;
549 GlobalValue *NsanShadowRetTag = nullptr;
550
551 Type *NsanShadowRetType = nullptr;
552 GlobalValue *NsanShadowRetPtr = nullptr;
553
554 GlobalValue *NsanShadowArgsTag = nullptr;
555
556 Type *NsanShadowArgsType = nullptr;
557 GlobalValue *NsanShadowArgsPtr = nullptr;
558
559 std::optional<Regex> CheckFunctionsFilter;
560};
561} // end anonymous namespace
562
563PreservedAnalyses
564NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
565 getOrCreateSanitizerCtorAndInitFunctions(
566 M, CtorName: kNsanModuleCtorName, InitName: kNsanInitName, /*InitArgTypes=*/{},
567 /*InitArgs=*/{},
568 // This callback is invoked when the functions are created the first
569 // time. Hook them into the global ctors list in that case:
570 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
571
572 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
573 NumericalStabilitySanitizer Nsan(Opts, M);
574 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
575 for (Function &F : M)
576 Nsan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
577
578 return PreservedAnalyses::none();
579}
580
581static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
582 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&M, Ty, Name] {
583 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
584 nullptr, Name, nullptr,
585 GlobalVariable::InitialExecTLSModel);
586 });
587}
588
589NumericalStabilitySanitizer::NumericalStabilitySanitizer(
590 const InstrumentationOptions &Opts, Module &M)
591 : Opts(Opts), DL(M.getDataLayout()), Context(M.getContext()),
592 Config(Opts, Context),
593 NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},
594 "__nsan_copy_values", /*NumArgs=*/3),
595 NsanSetUnknownFns(M,
596 {"__nsan_set_value_unknown_4",
597 "__nsan_set_value_unknown_8",
598 "__nsan_set_value_unknown_16"},
599 "__nsan_set_value_unknown", /*NumArgs=*/2) {
600 IntptrTy = DL.getIntPtrType(C&: Context);
601 Type *PtrTy = PointerType::getUnqual(C&: Context);
602 Type *Int32Ty = Type::getInt32Ty(C&: Context);
603 Type *Int1Ty = Type::getInt1Ty(C&: Context);
604 Type *VoidTy = Type::getVoidTy(C&: Context);
605
606 AttributeList Attr;
607 Attr = Attr.addFnAttribute(C&: Context, Kind: Attribute::NoUnwind);
608
609 Triple TT(M.getTargetTriple());
610 Attribute::AttrKind I32RetExtAttr =
611 TargetLibraryInfo::getExtAttrForI32Return(T: TT, /*Signed=*/true);
612 Attribute::AttrKind I32ArgExtAttr =
613 TargetLibraryInfo::getExtAttrForI32Param(T: TT, /*Signed=*/true);
614 Attribute::AttrKind BoolExtAttr =
615 TargetLibraryInfo::getExtAttrForBoolParam(T: TT);
616
617 // Initialize the runtime values (functions and global variables).
618 for (int I = 0; I < kNumValueTypes; ++I) {
619 const FTValueType VT = static_cast<FTValueType>(I);
620 const char *VTName = typeNameFromFTValueType(VT);
621 Type *VTTy = typeFromFTValueType(VT, Context);
622
623 // Load/store.
624 const std::string GetterPrefix =
625 std::string("__nsan_get_shadow_ptr_for_") + VTName;
626 NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
627 Name: GetterPrefix + "_store", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
628 NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
629 Name: GetterPrefix + "_load", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
630
631 // Check.
632 const auto &ShadowConfig = Config.byValueType(VT);
633 Type *ShadowTy = ShadowConfig.getType(Context);
634 {
635 AttributeList AL = Attr;
636 AL = AL.maybeAddRetAttribute(C&: Context, Kind: I32RetExtAttr);
637 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 2, Kind: I32ArgExtAttr);
638 NsanCheckValue[VT] =
639 M.getOrInsertFunction(Name: std::string("__nsan_internal_check_") + VTName +
640 "_" + ShadowConfig.getNsanTypeId(),
641 AttributeList: AL, RetTy: Int32Ty, Args: VTTy, Args: ShadowTy, Args: Int32Ty, Args: IntptrTy);
642 }
643
644 {
645 AttributeList AL = Attr;
646 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 4, Kind: I32ArgExtAttr);
647 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 5, Kind: BoolExtAttr);
648 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 6, Kind: BoolExtAttr);
649 NsanFCmpFail[VT] = M.getOrInsertFunction(
650 Name: std::string("__nsan_fcmp_fail_") + VTName + "_" +
651 ShadowConfig.getNsanTypeId(),
652 AttributeList: AL, RetTy: VoidTy, Args: VTTy, Args: VTTy, Args: ShadowTy, Args: ShadowTy, Args: Int32Ty, Args: Int1Ty, Args: Int1Ty);
653 }
654 }
655
656 // TODO: Add attributes nofree, nosync, readnone, readonly,
657 NsanGetRawShadowTypePtr = M.getOrInsertFunction(
658 Name: "__nsan_internal_get_raw_shadow_type_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
659 NsanGetRawShadowPtr = M.getOrInsertFunction(
660 Name: "__nsan_internal_get_raw_shadow_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
661
662 NsanShadowRetTag = createThreadLocalGV(Name: "__nsan_shadow_ret_tag", M, Ty: IntptrTy);
663
664 NsanShadowRetType = ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
665 NumElements: kMaxVectorWidth * kMaxShadowTypeSizeBytes);
666 NsanShadowRetPtr =
667 createThreadLocalGV(Name: "__nsan_shadow_ret_ptr", M, Ty: NsanShadowRetType);
668
669 NsanShadowArgsTag =
670 createThreadLocalGV(Name: "__nsan_shadow_args_tag", M, Ty: IntptrTy);
671
672 NsanShadowArgsType =
673 ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
674 NumElements: kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
675
676 NsanShadowArgsPtr =
677 createThreadLocalGV(Name: "__nsan_shadow_args_ptr", M, Ty: NsanShadowArgsType);
678
679 if (!Opts.check_functions_filter.empty()) {
680 Regex R = Regex(Opts.check_functions_filter);
681 std::string RegexError;
682 assert(R.isValid(RegexError));
683 CheckFunctionsFilter = std::move(R);
684 }
685}
686
687// Returns true if the given LLVM Value points to constant data (typically, a
688// global variable reference).
689bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
690 // If this is a GEP, just analyze its pointer operand.
691 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Addr))
692 Addr = GEP->getPointerOperand();
693
694 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr))
695 return GV->isConstant();
696 return false;
697}
698
699// This instruments the function entry to create shadow arguments.
700// Pseudocode:
701// if (this_fn_ptr == __nsan_shadow_args_tag) {
702// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
703// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
704// ...
705// __nsan_shadow_args_tag = 0;
706// } else {
707// s(arg0) = fext(arg0);
708// s(arg1) = fext(arg1);
709// ...
710// }
711void NumericalStabilitySanitizer::createShadowArguments(
712 Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
713 assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
714
715 // Do not bother if there are no FP args.
716 if (all_of(Range: F.args(), P: [this](const Argument &Arg) {
717 return Config.getExtendedFPType(FT: Arg.getType()) == nullptr;
718 }))
719 return;
720
721 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIIt());
722 // The function has shadow args if the shadow args tag matches the function
723 // address.
724 Value *HasShadowArgs = Builder.CreateICmpEQ(
725 LHS: Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowArgsTag, /*isVolatile=*/false),
726 RHS: Builder.CreatePtrToInt(V: &F, DestTy: IntptrTy));
727
728 unsigned ShadowArgsOffsetBytes = 0;
729 for (Argument &Arg : F.args()) {
730 Type *VT = Arg.getType();
731 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
732 if (ExtendedVT == nullptr)
733 continue; // Not an FT value.
734 Value *L = Builder.CreateAlignedLoad(
735 Ty: ExtendedVT,
736 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
737 Idx1: ShadowArgsOffsetBytes),
738 Align: Align(1), /*isVolatile=*/false);
739 Value *Shadow = Builder.CreateSelect(C: HasShadowArgs, True: L,
740 False: Builder.CreateFPExt(V: &Arg, DestTy: ExtendedVT));
741 Map.setShadow(V&: Arg, Shadow&: *Shadow);
742 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
743 assert(!SlotSize.isScalable() && "unsupported");
744 ShadowArgsOffsetBytes += SlotSize;
745 }
746 Builder.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: 0), Ptr: NsanShadowArgsTag);
747}
748
749// Returns true if the instrumentation should emit code to check arguments
750// before a function call.
751static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
752 const std::optional<Regex> &CheckFunctionsFilter) {
753
754 Function *Fn = CI.getCalledFunction();
755
756 if (CheckFunctionsFilter) {
757 // Skip checking args of indirect calls.
758 if (Fn == nullptr)
759 return false;
760 if (CheckFunctionsFilter->match(String: Fn->getName()))
761 return true;
762 return false;
763 }
764
765 if (Fn == nullptr)
766 return true; // Always check args of indirect calls.
767
768 // Never check nsan functions, the user called them for a reason.
769 if (Fn->getName().starts_with(Prefix: "__nsan_"))
770 return false;
771
772 const auto ID = Fn->getIntrinsicID();
773 LibFunc LFunc = TLI.getLibFunc(FDecl: *Fn);
774 // Always check args of unknown functions.
775 if (ID == Intrinsic::ID() && LFunc == NotLibFunc)
776 return true;
777
778 // Do not check args of an `fabs` call that is used for a comparison.
779 // This is typically used for `fabs(a-b) < tolerance`, where what matters is
780 // the result of the comparison, which is already caught be the fcmp checks.
781 if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
782 LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
783 for (const auto &U : CI.users())
784 if (isa<CmpInst>(Val: U))
785 return false;
786
787 return true; // Default is check.
788}
789
790// Populates the shadow call stack (which contains shadow values for every
791// floating-point parameter to the function).
792void NumericalStabilitySanitizer::populateShadowStack(
793 CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
794 // Do not create a shadow stack for inline asm.
795 if (CI.isInlineAsm())
796 return;
797
798 // Do not bother if there are no FP args.
799 if (all_of(Range: CI.operands(), P: [this](const Value *Arg) {
800 return Config.getExtendedFPType(FT: Arg->getType()) == nullptr;
801 }))
802 return;
803
804 IRBuilder<> Builder(&CI);
805 SmallVector<Value *, 8> ArgShadows;
806 const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
807 for (auto [ArgIdx, Arg] : enumerate(First: CI.operands())) {
808 if (Config.getExtendedFPType(FT: Arg->getType()) == nullptr)
809 continue; // Not an FT value.
810 Value *ArgShadow = Map.getShadow(V: Arg);
811 ArgShadows.push_back(Elt: ShouldCheckArgs ? emitCheck(V: Arg, ShadowV: ArgShadow, Builder,
812 Loc: CheckLoc::makeArg(ArgId: ArgIdx))
813 : ArgShadow);
814 }
815
816 // Do not create shadow stacks for intrinsics/known lib funcs.
817 if (Function *Fn = CI.getCalledFunction()) {
818 if (Fn->isIntrinsic() || TLI.getLibFunc(FDecl: *Fn) != NotLibFunc)
819 return;
820 }
821
822 // Set the shadow stack tag.
823 Builder.CreateStore(Val: CI.getCalledOperand(), Ptr: NsanShadowArgsTag);
824 TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(ExactSize: 0);
825
826 unsigned ShadowArgId = 0;
827 for (const Value *Arg : CI.operands()) {
828 Type *VT = Arg->getType();
829 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
830 if (ExtendedVT == nullptr)
831 continue; // Not an FT value.
832 Builder.CreateAlignedStore(
833 Val: ArgShadows[ShadowArgId++],
834 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
835 Idx1: ShadowArgsOffsetBytes),
836 Align: Align(1), /*isVolatile=*/false);
837 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
838 assert(!SlotSize.isScalable() && "unsupported");
839 ShadowArgsOffsetBytes += SlotSize;
840 }
841}
842
843// Internal part of emitCheck(). Returns a value that indicates whether
844// computation should continue with the shadow or resume by re-fextending the
845// value.
846enum class ContinuationType { // Keep in sync with runtime.
847 ContinueWithShadow = 0,
848 ResumeFromValue = 1,
849};
850
851Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
852 IRBuilder<> &Builder,
853 CheckLoc Loc) {
854 // Do not emit checks for constant values, this is redundant.
855 if (isa<Constant>(Val: V))
856 return ConstantInt::get(
857 Ty: Builder.getInt32Ty(),
858 V: static_cast<int>(ContinuationType::ContinueWithShadow));
859
860 Type *Ty = V->getType();
861 if (const auto VT = ftValueTypeFromType(FT: Ty))
862 return Builder.CreateCall(
863 Callee: NsanCheckValue[*VT],
864 Args: {V, ShadowV, Loc.getType(C&: Context), Loc.getValue(IntptrTy, Builder)});
865
866 if (Ty->isVectorTy()) {
867 auto *VecTy = cast<VectorType>(Val: Ty);
868 // We currently skip scalable vector types in MappingConfig,
869 // thus we should not encounter any such types here.
870 assert(!VecTy->isScalableTy() &&
871 "Scalable vector types are not supported yet");
872 Value *CheckResult = nullptr;
873 for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
874 // We resume if any element resumes. Another option would be to create a
875 // vector shuffle with the array of ContinueWithShadow, but that is too
876 // complex.
877 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
878 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
879 Value *ComponentCheckResult =
880 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
881 CheckResult = CheckResult
882 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
883 : ComponentCheckResult;
884 }
885 return CheckResult;
886 }
887 if (Ty->isArrayTy()) {
888 Value *CheckResult = nullptr;
889 for (auto I : seq(Size: Ty->getArrayNumElements())) {
890 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
891 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
892 Value *ComponentCheckResult =
893 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
894 CheckResult = CheckResult
895 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
896 : ComponentCheckResult;
897 }
898 return CheckResult;
899 }
900 if (Ty->isStructTy()) {
901 Value *CheckResult = nullptr;
902 for (auto I : seq(Size: Ty->getStructNumElements())) {
903 if (Config.getExtendedFPType(FT: Ty->getStructElementType(N: I)) == nullptr)
904 continue; // Only check FT values.
905 Value *ExtractV = Builder.CreateExtractValue(Agg: V, Idxs: I);
906 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
907 Value *ComponentCheckResult =
908 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
909 CheckResult = CheckResult
910 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
911 : ComponentCheckResult;
912 }
913 if (!CheckResult)
914 return ConstantInt::get(
915 Ty: Builder.getInt32Ty(),
916 V: static_cast<int>(ContinuationType::ContinueWithShadow));
917 return CheckResult;
918 }
919
920 llvm_unreachable("not implemented");
921}
922
923// Inserts a runtime check of V against its shadow value ShadowV.
924// We check values whenever they escape: on return, call, stores, and
925// insertvalue.
926// Returns the shadow value that should be used to continue the computations,
927// depending on the answer from the runtime.
928// TODO: Should we check on select ? phi ?
929Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
930 IRBuilder<> &Builder,
931 CheckLoc Loc) {
932 // Do not emit checks for constant values, this is redundant.
933 if (isa<Constant>(Val: V))
934 return ShadowV;
935
936 if (Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
937 Function *F = Inst->getFunction();
938 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName())) {
939 return ShadowV;
940 }
941 }
942
943 Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
944 Value *ICmpEQ = Builder.CreateICmpEQ(
945 LHS: CheckResult,
946 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
947 V: static_cast<int>(ContinuationType::ResumeFromValue)));
948 return Builder.CreateSelect(
949 C: ICmpEQ, True: Builder.CreateFPExt(V, DestTy: Config.getExtendedFPType(FT: V->getType())),
950 False: ShadowV);
951}
952
953// Inserts a check that fcmp on shadow values are consistent with that on base
954// values.
955void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
956 const ValueToShadowMap &Map) {
957 if (!Opts.nsan_instrument_fcmp)
958 return;
959
960 Function *F = FCmp.getFunction();
961 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName()))
962 return;
963
964 Value *LHS = FCmp.getOperand(i_nocapture: 0);
965 if (Config.getExtendedFPType(FT: LHS->getType()) == nullptr)
966 return;
967 Value *RHS = FCmp.getOperand(i_nocapture: 1);
968
969 // Split the basic block. On mismatch, we'll jump to the new basic block with
970 // a call to the runtime for error reporting.
971 BasicBlock *FCmpBB = FCmp.getParent();
972 BasicBlock *NextBB = FCmpBB->splitBasicBlock(I: FCmp.getNextNode());
973 // Remove the newly created terminator unconditional branch.
974 FCmpBB->back().eraseFromParent();
975 BasicBlock *FailBB =
976 BasicBlock::Create(Context, Name: "", Parent: FCmpBB->getParent(), InsertBefore: NextBB);
977
978 // Create the shadow fcmp and comparison between the fcmps.
979 IRBuilder<> FCmpBuilder(FCmpBB);
980 FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
981 Value *ShadowLHS = Map.getShadow(V: LHS);
982 Value *ShadowRHS = Map.getShadow(V: RHS);
983 // See the help text of -nsan-truncate-fcmp-eq.
984 if (FCmp.isEquality() && Opts.nsan_truncate_fcmp_eq) {
985 Type *Ty = ShadowLHS->getType();
986 ShadowLHS = FCmpBuilder.CreateFPExt(
987 V: FCmpBuilder.CreateFPTrunc(V: ShadowLHS, DestTy: LHS->getType()), DestTy: Ty);
988 ShadowRHS = FCmpBuilder.CreateFPExt(
989 V: FCmpBuilder.CreateFPTrunc(V: ShadowRHS, DestTy: RHS->getType()), DestTy: Ty);
990 }
991 Value *ShadowFCmp =
992 FCmpBuilder.CreateFCmp(P: FCmp.getPredicate(), LHS: ShadowLHS, RHS: ShadowRHS);
993 Value *OriginalAndShadowFcmpMatch =
994 FCmpBuilder.CreateICmpEQ(LHS: &FCmp, RHS: ShadowFCmp);
995
996 if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
997 // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
998 // where an element is true if the corresponding elements in original and
999 // shadow are the same. We want all elements to be 1.
1000 OriginalAndShadowFcmpMatch =
1001 FCmpBuilder.CreateAndReduce(Src: OriginalAndShadowFcmpMatch);
1002 }
1003
1004 // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common
1005 // case.
1006 FCmpBuilder.CreateCondBr(Cond: OriginalAndShadowFcmpMatch, True: NextBB, False: FailBB,
1007 BranchWeights: MDBuilder(Context).createLikelyBranchWeights());
1008
1009 // Fill in FailBB.
1010 IRBuilder<> FailBuilder(FailBB);
1011 FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1012
1013 const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
1014 &FailBuilder](Value *L, Value *R, Value *ShadowL,
1015 Value *ShadowR, Value *Result,
1016 Value *ShadowResult) {
1017 Type *FT = L->getType();
1018 FunctionCallee *Callee = nullptr;
1019 if (FT->isFloatTy()) {
1020 Callee = &(NsanFCmpFail[kFloat]);
1021 } else if (FT->isDoubleTy()) {
1022 Callee = &(NsanFCmpFail[kDouble]);
1023 } else if (FT->isX86_FP80Ty()) {
1024 // TODO: make NsanFCmpFailLongDouble work.
1025 Callee = &(NsanFCmpFail[kDouble]);
1026 L = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1027 R = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1028 } else {
1029 llvm_unreachable("not implemented");
1030 }
1031 FailBuilder.CreateCall(Callee: *Callee, Args: {L, R, ShadowL, ShadowR,
1032 ConstantInt::get(Ty: FCmpBuilder.getInt32Ty(),
1033 V: FCmp.getPredicate()),
1034 Result, ShadowResult});
1035 };
1036 if (LHS->getType()->isVectorTy()) {
1037 for (int I = 0, E = cast<VectorType>(Val: LHS->getType())
1038 ->getElementCount()
1039 .getFixedValue();
1040 I < E; ++I) {
1041 Value *ExtractLHS = FailBuilder.CreateExtractElement(Vec: LHS, Idx: I);
1042 Value *ExtractRHS = FailBuilder.CreateExtractElement(Vec: RHS, Idx: I);
1043 Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(Vec: ShadowLHS, Idx: I);
1044 Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(Vec: ShadowRHS, Idx: I);
1045 Value *ExtractFCmp = FailBuilder.CreateExtractElement(Vec: &FCmp, Idx: I);
1046 Value *ExtractShadowFCmp =
1047 FailBuilder.CreateExtractElement(Vec: ShadowFCmp, Idx: I);
1048 EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,
1049 ExtractFCmp, ExtractShadowFCmp);
1050 }
1051 } else {
1052 EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
1053 }
1054 FailBuilder.CreateBr(Dest: NextBB);
1055
1056 ++NumInstrumentedFCmp;
1057}
1058
1059// Creates a shadow phi value for any phi that defines a value of FT type.
1060PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
1061 PHINode &Phi, const TargetLibraryInfo &TLI) {
1062 Type *VT = Phi.getType();
1063 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1064 if (ExtendedVT == nullptr)
1065 return nullptr; // Not an FT value.
1066 // The phi operands are shadow values and are not available when the phi is
1067 // created. They will be populated in a final phase, once all shadow values
1068 // have been created.
1069 PHINode *Shadow = PHINode::Create(Ty: ExtendedVT, NumReservedValues: Phi.getNumIncomingValues());
1070 Shadow->insertAfter(InsertPos: Phi.getIterator());
1071 return Shadow;
1072}
1073
1074Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
1075 Type *ExtendedVT) {
1076 IRBuilder<> Builder(Load.getNextNode());
1077 Builder.SetCurrentDebugLocation(Load.getDebugLoc());
1078 if (addrPointsToConstantData(Addr: Load.getPointerOperand())) {
1079 // No need to look into the shadow memory, the value is a constant. Just
1080 // convert from FT to 2FT.
1081 return Builder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1082 }
1083
1084 // if (%shadowptr == &)
1085 // %shadow = fpext %v
1086 // else
1087 // %shadow = load (ptrcast %shadow_ptr))
1088 // Considered options here:
1089 // - Have `NsanGetShadowPtrForLoad` return a fixed address
1090 // &__nsan_unknown_value_shadow_address that is valid to load from, and
1091 // use a select. This has the advantage that the generated IR is simpler.
1092 // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does
1093 // not short-circuit, dereferencing the returned pointer is no longer an
1094 // option, have to split and create a separate basic block. This has the
1095 // advantage of being easier to debug because it crashes if we ever mess
1096 // up.
1097
1098 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1099 Value *ShadowPtr = Builder.CreateCall(
1100 Callee: NsanGetShadowPtrForLoad[Extents.ValueType],
1101 Args: {Load.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1102 ++NumInstrumentedFTLoads;
1103
1104 // Split the basic block.
1105 BasicBlock *LoadBB = Load.getParent();
1106 BasicBlock *NextBB = LoadBB->splitBasicBlock(I: Builder.GetInsertPoint());
1107 // Create the two options for creating the shadow value.
1108 BasicBlock *ShadowLoadBB =
1109 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1110 BasicBlock *FExtBB =
1111 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1112
1113 // Replace the newly created terminator unconditional branch by a conditional
1114 // branch to one of the options.
1115 {
1116 LoadBB->back().eraseFromParent();
1117 IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
1118 LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1119 LoadBBBuilder.CreateCondBr(Cond: LoadBBBuilder.CreateIsNull(Arg: ShadowPtr), True: FExtBB,
1120 False: ShadowLoadBB);
1121 }
1122
1123 // Fill in ShadowLoadBB.
1124 IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
1125 ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1126 Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
1127 Ty: ExtendedVT, Ptr: ShadowPtr, Align: Align(1), isVolatile: Load.isVolatile());
1128 if (Opts.nsan_check_loads) {
1129 ShadowLoad = emitCheck(V: &Load, ShadowV: ShadowLoad, Builder&: ShadowLoadBBBuilder,
1130 Loc: CheckLoc::makeLoad(Address: Load.getPointerOperand()));
1131 }
1132 ShadowLoadBBBuilder.CreateBr(Dest: NextBB);
1133
1134 // Fill in FExtBB.
1135 IRBuilder<> FExtBBBuilder(FExtBB);
1136 FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1137 Value *FExt = FExtBBBuilder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1138 FExtBBBuilder.CreateBr(Dest: NextBB);
1139
1140 // The shadow value come from any of the options.
1141 IRBuilder<> NextBBBuilder(&*NextBB->begin());
1142 NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1143 PHINode *ShadowPhi = NextBBBuilder.CreatePHI(Ty: ExtendedVT, NumReservedValues: 2);
1144 ShadowPhi->addIncoming(V: ShadowLoad, BB: ShadowLoadBB);
1145 ShadowPhi->addIncoming(V: FExt, BB: FExtBB);
1146 return ShadowPhi;
1147}
1148
1149Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,
1150 Type *VT, Type *ExtendedVT,
1151 const ValueToShadowMap &Map,
1152 IRBuilder<> &Builder) {
1153 Value *OrigSource = Trunc.getOperand(i_nocapture: 0);
1154 Type *OrigSourceTy = OrigSource->getType();
1155 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1156
1157 // When truncating:
1158 // - (A) If the source has a shadow, we truncate from the shadow, else we
1159 // truncate from the original source.
1160 // - (B) If the shadow of the source is larger than the shadow of the dest,
1161 // we still need a truncate. Else, the shadow of the source is the same
1162 // type as the shadow of the dest (because mappings are non-decreasing), so
1163 // we don't need to emit a truncate.
1164 // Examples,
1165 // with a mapping of {f32->f64;f64->f80;f80->f128}
1166 // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double
1167 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1168 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1169 // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1)
1170 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80
1171 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1172 // with a mapping of {f32->f64;f64->f128;f80->f128}
1173 // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double
1174 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1175 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1176 // fptrunc x86_fp80 %1 to double -> fp128 %1
1177 // fptrunc fp128 %1 to double -> fp128 %1
1178 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1179 // with a mapping of {f32->f32;f64->f32;f80->f64}
1180 // fptrunc double %1 to float -> float s(%1)
1181 // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float
1182 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float
1183 // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float
1184 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float
1185 // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double
1186
1187 // See (A) above.
1188 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1189 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1190 // See (B) above.
1191 if (SourceTy == ExtendedVT)
1192 return Source;
1193
1194 return Builder.CreateFPTrunc(V: Source, DestTy: ExtendedVT);
1195}
1196
1197Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,
1198 Type *ExtendedVT,
1199 const ValueToShadowMap &Map,
1200 IRBuilder<> &Builder) {
1201 Value *OrigSource = Ext.getOperand(i_nocapture: 0);
1202 Type *OrigSourceTy = OrigSource->getType();
1203 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1204 // When extending:
1205 // - (A) If the source has a shadow, we extend from the shadow, else we
1206 // extend from the original source.
1207 // - (B) If the shadow of the dest is larger than the shadow of the source,
1208 // we still need an extend. Else, the shadow of the source is the same
1209 // type as the shadow of the dest (because mappings are non-decreasing), so
1210 // we don't need to emit an extend.
1211 // Examples,
1212 // with a mapping of {f32->f64;f64->f80;f80->f128}
1213 // fpext half %1 to float -> fpext half %1 to double
1214 // fpext half %1 to double -> fpext half %1 to x86_fp80
1215 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1216 // fpext float %1 to double -> double s(%1)
1217 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1218 // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128
1219 // with a mapping of {f32->f64;f64->f128;f80->f128}
1220 // fpext half %1 to float -> fpext half %1 to double
1221 // fpext half %1 to double -> fpext half %1 to fp128
1222 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1223 // fpext float %1 to double -> fpext double s(%1) to fp128
1224 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1225 // fpext double %1 to x86_fp80 -> fp128 s(%1)
1226 // with a mapping of {f32->f32;f64->f32;f80->f64}
1227 // fpext half %1 to float -> fpext half %1 to float
1228 // fpext half %1 to double -> fpext half %1 to float
1229 // fpext half %1 to x86_fp80 -> fpext half %1 to double
1230 // fpext float %1 to double -> s(%1)
1231 // fpext float %1 to x86_fp80 -> fpext float s(%1) to double
1232 // fpext double %1 to x86_fp80 -> fpext float s(%1) to double
1233
1234 // See (A) above.
1235 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1236 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1237 // See (B) above.
1238 if (SourceTy == ExtendedVT)
1239 return Source;
1240
1241 return Builder.CreateFPExt(V: Source, DestTy: ExtendedVT);
1242}
1243
1244namespace {
1245// TODO: This should be tablegen-ed.
1246struct KnownIntrinsic {
1247 struct WidenedIntrinsic {
1248 const char *NarrowName;
1249 Intrinsic::ID ID; // wide id.
1250 using FnTypeFactory = FunctionType *(*)(LLVMContext &);
1251 FnTypeFactory MakeFnTy;
1252 };
1253
1254 static const char *get(LibFunc LFunc);
1255
1256 // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
1257 // that applies the same operation on the shadow argument.
1258 // Options are:
1259 // - pass in the ID and full function type,
1260 // - pass in the name, which includes the function type through mangling.
1261 static const WidenedIntrinsic *widen(StringRef Name);
1262
1263private:
1264 struct LFEntry {
1265 LibFunc LFunc;
1266 const char *IntrinsicName;
1267 };
1268 static const LFEntry kLibfuncIntrinsics[];
1269
1270 static const WidenedIntrinsic kWidenedIntrinsics[];
1271};
1272} // namespace
1273
1274static FunctionType *makeDoubleDouble(LLVMContext &C) {
1275 return FunctionType::get(Result: Type::getDoubleTy(C), Params: {Type::getDoubleTy(C)}, isVarArg: false);
1276}
1277
1278static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {
1279 return FunctionType::get(Result: Type::getX86_FP80Ty(C), Params: {Type::getX86_FP80Ty(C)},
1280 isVarArg: false);
1281}
1282
1283static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {
1284 return FunctionType::get(Result: Type::getDoubleTy(C),
1285 Params: {Type::getDoubleTy(C), Type::getInt32Ty(C)}, isVarArg: false);
1286}
1287
1288static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {
1289 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1290 Params: {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
1291 isVarArg: false);
1292}
1293
1294static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {
1295 return FunctionType::get(Result: Type::getDoubleTy(C),
1296 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C)}, isVarArg: false);
1297}
1298
1299static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {
1300 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1301 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1302 isVarArg: false);
1303}
1304
1305static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {
1306 return FunctionType::get(
1307 Result: Type::getDoubleTy(C),
1308 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
1309 isVarArg: false);
1310}
1311
1312static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {
1313 return FunctionType::get(
1314 Result: Type::getX86_FP80Ty(C),
1315 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1316 isVarArg: false);
1317}
1318
1319const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
1320 // TODO: Right now we ignore vector intrinsics.
1321 // This is hard because we have to model the semantics of the intrinsics,
1322 // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
1323 // Intrinsics that take any non-vector FT types:
1324 // NOTE: Right now because of
1325 // https://github.com/llvm/llvm-project/issues/44744
1326 // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and
1327 // come back).
1328 {.NarrowName: "llvm.sqrt.f32", .ID: Intrinsic::sqrt, .MakeFnTy: makeDoubleDouble},
1329 {.NarrowName: "llvm.sqrt.f64", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1330 {.NarrowName: "llvm.sqrt.f80", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1331 {.NarrowName: "llvm.powi.f32", .ID: Intrinsic::powi, .MakeFnTy: makeDoubleDoubleI32},
1332 {.NarrowName: "llvm.powi.f64", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1333 {.NarrowName: "llvm.powi.f80", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1334 {.NarrowName: "llvm.sin.f32", .ID: Intrinsic::sin, .MakeFnTy: makeDoubleDouble},
1335 {.NarrowName: "llvm.sin.f64", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1336 {.NarrowName: "llvm.sin.f80", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1337 {.NarrowName: "llvm.cos.f32", .ID: Intrinsic::cos, .MakeFnTy: makeDoubleDouble},
1338 {.NarrowName: "llvm.cos.f64", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1339 {.NarrowName: "llvm.cos.f80", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1340 {.NarrowName: "llvm.pow.f32", .ID: Intrinsic::pow, .MakeFnTy: makeDoubleDoubleDouble},
1341 {.NarrowName: "llvm.pow.f64", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1342 {.NarrowName: "llvm.pow.f80", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1343 {.NarrowName: "llvm.exp.f32", .ID: Intrinsic::exp, .MakeFnTy: makeDoubleDouble},
1344 {.NarrowName: "llvm.exp.f64", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1345 {.NarrowName: "llvm.exp.f80", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1346 {.NarrowName: "llvm.exp2.f32", .ID: Intrinsic::exp2, .MakeFnTy: makeDoubleDouble},
1347 {.NarrowName: "llvm.exp2.f64", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1348 {.NarrowName: "llvm.exp2.f80", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1349 {.NarrowName: "llvm.log.f32", .ID: Intrinsic::log, .MakeFnTy: makeDoubleDouble},
1350 {.NarrowName: "llvm.log.f64", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1351 {.NarrowName: "llvm.log.f80", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1352 {.NarrowName: "llvm.log10.f32", .ID: Intrinsic::log10, .MakeFnTy: makeDoubleDouble},
1353 {.NarrowName: "llvm.log10.f64", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1354 {.NarrowName: "llvm.log10.f80", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1355 {.NarrowName: "llvm.log2.f32", .ID: Intrinsic::log2, .MakeFnTy: makeDoubleDouble},
1356 {.NarrowName: "llvm.log2.f64", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1357 {.NarrowName: "llvm.log2.f80", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1358 {.NarrowName: "llvm.fma.f32", .ID: Intrinsic::fma, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1359 {.NarrowName: "llvm.fma.f64", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1360 {.NarrowName: "llvm.fma.f80", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1361 {.NarrowName: "llvm.fmuladd.f32", .ID: Intrinsic::fmuladd, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1362 {.NarrowName: "llvm.fmuladd.f64", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1363 {.NarrowName: "llvm.fmuladd.f80", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1364 {.NarrowName: "llvm.fabs.f32", .ID: Intrinsic::fabs, .MakeFnTy: makeDoubleDouble},
1365 {.NarrowName: "llvm.fabs.f64", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1366 {.NarrowName: "llvm.fabs.f80", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1367 {.NarrowName: "llvm.minnum.f32", .ID: Intrinsic::minnum, .MakeFnTy: makeDoubleDoubleDouble},
1368 {.NarrowName: "llvm.minnum.f64", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1369 {.NarrowName: "llvm.minnum.f80", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1370 {.NarrowName: "llvm.maxnum.f32", .ID: Intrinsic::maxnum, .MakeFnTy: makeDoubleDoubleDouble},
1371 {.NarrowName: "llvm.maxnum.f64", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1372 {.NarrowName: "llvm.maxnum.f80", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1373 {.NarrowName: "llvm.minimum.f32", .ID: Intrinsic::minimum, .MakeFnTy: makeDoubleDoubleDouble},
1374 {.NarrowName: "llvm.minimum.f64", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1375 {.NarrowName: "llvm.minimum.f80", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1376 {.NarrowName: "llvm.maximum.f32", .ID: Intrinsic::maximum, .MakeFnTy: makeDoubleDoubleDouble},
1377 {.NarrowName: "llvm.maximum.f64", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1378 {.NarrowName: "llvm.maximum.f80", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1379 {.NarrowName: "llvm.minimumnum.f32", .ID: Intrinsic::minimumnum, .MakeFnTy: makeDoubleDoubleDouble},
1380 {.NarrowName: "llvm.minimumnum.f64", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1381 {.NarrowName: "llvm.minimumnum.f80", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1382 {.NarrowName: "llvm.maximumnum.f32", .ID: Intrinsic::maximumnum, .MakeFnTy: makeDoubleDoubleDouble},
1383 {.NarrowName: "llvm.maximumnum.f64", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1384 {.NarrowName: "llvm.maximumnum.f80", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1385 {.NarrowName: "llvm.copysign.f32", .ID: Intrinsic::copysign, .MakeFnTy: makeDoubleDoubleDouble},
1386 {.NarrowName: "llvm.copysign.f64", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1387 {.NarrowName: "llvm.copysign.f80", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1388 {.NarrowName: "llvm.floor.f32", .ID: Intrinsic::floor, .MakeFnTy: makeDoubleDouble},
1389 {.NarrowName: "llvm.floor.f64", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1390 {.NarrowName: "llvm.floor.f80", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1391 {.NarrowName: "llvm.ceil.f32", .ID: Intrinsic::ceil, .MakeFnTy: makeDoubleDouble},
1392 {.NarrowName: "llvm.ceil.f64", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1393 {.NarrowName: "llvm.ceil.f80", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1394 {.NarrowName: "llvm.trunc.f32", .ID: Intrinsic::trunc, .MakeFnTy: makeDoubleDouble},
1395 {.NarrowName: "llvm.trunc.f64", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1396 {.NarrowName: "llvm.trunc.f80", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1397 {.NarrowName: "llvm.rint.f32", .ID: Intrinsic::rint, .MakeFnTy: makeDoubleDouble},
1398 {.NarrowName: "llvm.rint.f64", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1399 {.NarrowName: "llvm.rint.f80", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1400 {.NarrowName: "llvm.nearbyint.f32", .ID: Intrinsic::nearbyint, .MakeFnTy: makeDoubleDouble},
1401 {.NarrowName: "llvm.nearbyint.f64", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1402 {.NarrowName: "llvm.nearbyint.f80", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1403 {.NarrowName: "llvm.round.f32", .ID: Intrinsic::round, .MakeFnTy: makeDoubleDouble},
1404 {.NarrowName: "llvm.round.f64", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1405 {.NarrowName: "llvm.round.f80", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1406};
1407
1408const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {
1409 {.LFunc: LibFunc_sqrtf, .IntrinsicName: "llvm.sqrt.f32"},
1410 {.LFunc: LibFunc_sqrt, .IntrinsicName: "llvm.sqrt.f64"},
1411 {.LFunc: LibFunc_sqrtl, .IntrinsicName: "llvm.sqrt.f80"},
1412 {.LFunc: LibFunc_sinf, .IntrinsicName: "llvm.sin.f32"},
1413 {.LFunc: LibFunc_sin, .IntrinsicName: "llvm.sin.f64"},
1414 {.LFunc: LibFunc_sinl, .IntrinsicName: "llvm.sin.f80"},
1415 {.LFunc: LibFunc_cosf, .IntrinsicName: "llvm.cos.f32"},
1416 {.LFunc: LibFunc_cos, .IntrinsicName: "llvm.cos.f64"},
1417 {.LFunc: LibFunc_cosl, .IntrinsicName: "llvm.cos.f80"},
1418 {.LFunc: LibFunc_powf, .IntrinsicName: "llvm.pow.f32"},
1419 {.LFunc: LibFunc_pow, .IntrinsicName: "llvm.pow.f64"},
1420 {.LFunc: LibFunc_powl, .IntrinsicName: "llvm.pow.f80"},
1421 {.LFunc: LibFunc_expf, .IntrinsicName: "llvm.exp.f32"},
1422 {.LFunc: LibFunc_exp, .IntrinsicName: "llvm.exp.f64"},
1423 {.LFunc: LibFunc_expl, .IntrinsicName: "llvm.exp.f80"},
1424 {.LFunc: LibFunc_exp2f, .IntrinsicName: "llvm.exp2.f32"},
1425 {.LFunc: LibFunc_exp2, .IntrinsicName: "llvm.exp2.f64"},
1426 {.LFunc: LibFunc_exp2l, .IntrinsicName: "llvm.exp2.f80"},
1427 {.LFunc: LibFunc_logf, .IntrinsicName: "llvm.log.f32"},
1428 {.LFunc: LibFunc_log, .IntrinsicName: "llvm.log.f64"},
1429 {.LFunc: LibFunc_logl, .IntrinsicName: "llvm.log.f80"},
1430 {.LFunc: LibFunc_log10f, .IntrinsicName: "llvm.log10.f32"},
1431 {.LFunc: LibFunc_log10, .IntrinsicName: "llvm.log10.f64"},
1432 {.LFunc: LibFunc_log10l, .IntrinsicName: "llvm.log10.f80"},
1433 {.LFunc: LibFunc_log2f, .IntrinsicName: "llvm.log2.f32"},
1434 {.LFunc: LibFunc_log2, .IntrinsicName: "llvm.log2.f64"},
1435 {.LFunc: LibFunc_log2l, .IntrinsicName: "llvm.log2.f80"},
1436 {.LFunc: LibFunc_fabsf, .IntrinsicName: "llvm.fabs.f32"},
1437 {.LFunc: LibFunc_fabs, .IntrinsicName: "llvm.fabs.f64"},
1438 {.LFunc: LibFunc_fabsl, .IntrinsicName: "llvm.fabs.f80"},
1439 {.LFunc: LibFunc_copysignf, .IntrinsicName: "llvm.copysign.f32"},
1440 {.LFunc: LibFunc_copysign, .IntrinsicName: "llvm.copysign.f64"},
1441 {.LFunc: LibFunc_copysignl, .IntrinsicName: "llvm.copysign.f80"},
1442 {.LFunc: LibFunc_floorf, .IntrinsicName: "llvm.floor.f32"},
1443 {.LFunc: LibFunc_floor, .IntrinsicName: "llvm.floor.f64"},
1444 {.LFunc: LibFunc_floorl, .IntrinsicName: "llvm.floor.f80"},
1445 {.LFunc: LibFunc_fmaxf, .IntrinsicName: "llvm.maxnum.f32"},
1446 {.LFunc: LibFunc_fmax, .IntrinsicName: "llvm.maxnum.f64"},
1447 {.LFunc: LibFunc_fmaxl, .IntrinsicName: "llvm.maxnum.f80"},
1448 {.LFunc: LibFunc_fminf, .IntrinsicName: "llvm.minnum.f32"},
1449 {.LFunc: LibFunc_fmin, .IntrinsicName: "llvm.minnum.f64"},
1450 {.LFunc: LibFunc_fminl, .IntrinsicName: "llvm.minnum.f80"},
1451 {.LFunc: LibFunc_fmaximum_numf, .IntrinsicName: "llvm.maximumnum.f32"},
1452 {.LFunc: LibFunc_fmaximum_num, .IntrinsicName: "llvm.maximumnum.f64"},
1453 {.LFunc: LibFunc_fmaximum_numl, .IntrinsicName: "llvm.maximumnum.f80"},
1454 {.LFunc: LibFunc_fminimum_numf, .IntrinsicName: "llvm.minimumnum.f32"},
1455 {.LFunc: LibFunc_fminimum_num, .IntrinsicName: "llvm.minimumnum.f64"},
1456 {.LFunc: LibFunc_fminimum_numl, .IntrinsicName: "llvm.minimumnum.f80"},
1457 {.LFunc: LibFunc_ceilf, .IntrinsicName: "llvm.ceil.f32"},
1458 {.LFunc: LibFunc_ceil, .IntrinsicName: "llvm.ceil.f64"},
1459 {.LFunc: LibFunc_ceill, .IntrinsicName: "llvm.ceil.f80"},
1460 {.LFunc: LibFunc_truncf, .IntrinsicName: "llvm.trunc.f32"},
1461 {.LFunc: LibFunc_trunc, .IntrinsicName: "llvm.trunc.f64"},
1462 {.LFunc: LibFunc_truncl, .IntrinsicName: "llvm.trunc.f80"},
1463 {.LFunc: LibFunc_rintf, .IntrinsicName: "llvm.rint.f32"},
1464 {.LFunc: LibFunc_rint, .IntrinsicName: "llvm.rint.f64"},
1465 {.LFunc: LibFunc_rintl, .IntrinsicName: "llvm.rint.f80"},
1466 {.LFunc: LibFunc_nearbyintf, .IntrinsicName: "llvm.nearbyint.f32"},
1467 {.LFunc: LibFunc_nearbyint, .IntrinsicName: "llvm.nearbyint.f64"},
1468 {.LFunc: LibFunc_nearbyintl, .IntrinsicName: "llvm.nearbyint.f80"},
1469 {.LFunc: LibFunc_roundf, .IntrinsicName: "llvm.round.f32"},
1470 {.LFunc: LibFunc_round, .IntrinsicName: "llvm.round.f64"},
1471 {.LFunc: LibFunc_roundl, .IntrinsicName: "llvm.round.f80"},
1472};
1473
1474const char *KnownIntrinsic::get(LibFunc LFunc) {
1475 for (const auto &E : kLibfuncIntrinsics) {
1476 if (E.LFunc == LFunc)
1477 return E.IntrinsicName;
1478 }
1479 return nullptr;
1480}
1481
1482const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {
1483 for (const auto &E : kWidenedIntrinsics) {
1484 if (E.NarrowName == Name)
1485 return &E;
1486 }
1487 return nullptr;
1488}
1489
1490// Returns the name of the LLVM intrinsic corresponding to the given function.
1491static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,
1492 const TargetLibraryInfo &TLI) {
1493 LibFunc LFunc = TLI.getLibFunc(FDecl: Fn);
1494 if (LFunc == NotLibFunc)
1495 return nullptr;
1496
1497 if (const char *Name = KnownIntrinsic::get(LFunc))
1498 return Name;
1499
1500 LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");
1501 return nullptr;
1502}
1503
1504// Try to handle a known function call.
1505Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(
1506 CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,
1507 const ValueToShadowMap &Map, IRBuilder<> &Builder) {
1508 Function *Fn = Call.getCalledFunction();
1509 if (Fn == nullptr)
1510 return nullptr;
1511
1512 Intrinsic::ID WidenedId = Intrinsic::ID();
1513 FunctionType *WidenedFnTy = nullptr;
1514 if (const auto ID = Fn->getIntrinsicID()) {
1515 const auto *Widened = KnownIntrinsic::widen(Name: Fn->getName());
1516 if (Widened) {
1517 WidenedId = Widened->ID;
1518 WidenedFnTy = Widened->MakeFnTy(Context);
1519 } else {
1520 // If we don't know how to widen the intrinsic, we have no choice but to
1521 // call the non-wide version on a truncated shadow and extend again
1522 // afterwards.
1523 WidenedId = ID;
1524 WidenedFnTy = Fn->getFunctionType();
1525 }
1526 } else if (const char *Name = getIntrinsicFromLibfunc(Fn&: *Fn, VT, TLI)) {
1527 // We might have a call to a library function that we can replace with a
1528 // wider Intrinsic.
1529 const auto *Widened = KnownIntrinsic::widen(Name);
1530 assert(Widened && "make sure KnownIntrinsic entries are consistent");
1531 WidenedId = Widened->ID;
1532 WidenedFnTy = Widened->MakeFnTy(Context);
1533 } else {
1534 // This is not a known library function or intrinsic.
1535 return nullptr;
1536 }
1537
1538 // Check that the widened intrinsic is valid.
1539 SmallVector<Type *, 4> OverloadTys;
1540 [[maybe_unused]] bool IsValid =
1541 Intrinsic::isSignatureValid(ID: WidenedId, FT: WidenedFnTy, OverloadTys);
1542 assert(IsValid && "invalid widened intrinsic");
1543 // For known intrinsic functions, we create a second call to the same
1544 // intrinsic with a different type.
1545 SmallVector<Value *, 4> Args;
1546 // The last operand is the intrinsic itself, skip it.
1547 for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {
1548 Value *Arg = Call.getOperand(i_nocapture: I);
1549 Type *OrigArgTy = Arg->getType();
1550 Type *IntrinsicArgTy = WidenedFnTy->getParamType(i: I);
1551 if (OrigArgTy == IntrinsicArgTy) {
1552 Args.push_back(Elt: Arg); // The arg is passed as is.
1553 continue;
1554 }
1555 Type *ShadowArgTy = Config.getExtendedFPType(FT: Arg->getType());
1556 assert(ShadowArgTy &&
1557 "don't know how to get the shadow value for a non-FT");
1558 Value *Shadow = Map.getShadow(V: Arg);
1559 if (ShadowArgTy == IntrinsicArgTy) {
1560 // The shadow is the right type for the intrinsic.
1561 assert(Shadow->getType() == ShadowArgTy);
1562 Args.push_back(Elt: Shadow);
1563 continue;
1564 }
1565 // There is no intrinsic with his level of precision, truncate the shadow.
1566 Args.push_back(Elt: Builder.CreateFPTrunc(V: Shadow, DestTy: IntrinsicArgTy));
1567 }
1568 Value *IntrinsicCall = Builder.CreateIntrinsic(ID: WidenedId, OverloadTypes: OverloadTys, Args);
1569 return WidenedFnTy->getReturnType() == ExtendedVT
1570 ? IntrinsicCall
1571 : Builder.CreateFPExt(V: IntrinsicCall, DestTy: ExtendedVT);
1572}
1573
1574// Handle a CallBase, i.e. a function call, an inline asm sequence, or an
1575// invoke.
1576Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,
1577 Type *ExtendedVT,
1578 const TargetLibraryInfo &TLI,
1579 const ValueToShadowMap &Map,
1580 IRBuilder<> &Builder) {
1581 // We cannot look inside inline asm, just expand the result again.
1582 if (Call.isInlineAsm())
1583 return Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT);
1584
1585 // Intrinsics and library functions (e.g. sin, exp) are handled
1586 // specifically, because we know their semantics and can do better than
1587 // blindly calling them (e.g. compute the sinus in the actual shadow domain).
1588 if (Value *V =
1589 maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))
1590 return V;
1591
1592 // If the return tag matches that of the called function, read the extended
1593 // return value from the shadow ret ptr. Else, just extend the return value.
1594 Value *L =
1595 Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowRetTag, /*isVolatile=*/false);
1596 Value *HasShadowRet = Builder.CreateICmpEQ(
1597 LHS: L, RHS: Builder.CreatePtrToInt(V: Call.getCalledOperand(), DestTy: IntptrTy));
1598
1599 Value *ShadowRetVal = Builder.CreateLoad(
1600 Ty: ExtendedVT,
1601 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0),
1602 /*isVolatile=*/false);
1603 Value *Shadow = Builder.CreateSelect(C: HasShadowRet, True: ShadowRetVal,
1604 False: Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT));
1605 ++NumInstrumentedFTCalls;
1606 return Shadow;
1607}
1608
1609// Creates a shadow value for the given FT value. At that point all operands are
1610// guaranteed to be available.
1611Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(
1612 Instruction &Inst, const TargetLibraryInfo &TLI,
1613 const ValueToShadowMap &Map) {
1614 Type *VT = Inst.getType();
1615 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1616 assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");
1617
1618 if (auto *Load = dyn_cast<LoadInst>(Val: &Inst))
1619 return handleLoad(Load&: *Load, VT, ExtendedVT);
1620
1621 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
1622 // Insert after the call.
1623 BasicBlock::iterator It(Inst);
1624 IRBuilder<> Builder(++It);
1625 Builder.SetCurrentDebugLocation(Call->getDebugLoc());
1626 return handleCallBase(Call&: *Call, VT, ExtendedVT, TLI, Map, Builder);
1627 }
1628
1629 if (auto *Invoke = dyn_cast<InvokeInst>(Val: &Inst)) {
1630 // The Invoke terminates the basic block, create a new basic block in
1631 // between the successful invoke and the next block.
1632 BasicBlock *InvokeBB = Invoke->getParent();
1633 BasicBlock *NextBB = Invoke->getNormalDest();
1634 BasicBlock *NewBB =
1635 BasicBlock::Create(Context, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
1636 Inst.replaceSuccessorWith(OldBB: NextBB, NewBB);
1637
1638 IRBuilder<> Builder(NewBB);
1639 Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());
1640 Value *Shadow = handleCallBase(Call&: *Invoke, VT, ExtendedVT, TLI, Map, Builder);
1641 Builder.CreateBr(Dest: NextBB);
1642 NewBB->replaceSuccessorsPhiUsesWith(Old: InvokeBB, New: NewBB);
1643 return Shadow;
1644 }
1645
1646 IRBuilder<> Builder(Inst.getNextNode());
1647 Builder.SetCurrentDebugLocation(Inst.getDebugLoc());
1648
1649 if (auto *Trunc = dyn_cast<FPTruncInst>(Val: &Inst))
1650 return handleTrunc(Trunc: *Trunc, VT, ExtendedVT, Map, Builder);
1651 if (auto *Ext = dyn_cast<FPExtInst>(Val: &Inst))
1652 return handleExt(Ext: *Ext, VT, ExtendedVT, Map, Builder);
1653
1654 if (auto *UnaryOp = dyn_cast<UnaryOperator>(Val: &Inst))
1655 return Builder.CreateUnOp(Opc: UnaryOp->getOpcode(),
1656 V: Map.getShadow(V: UnaryOp->getOperand(i_nocapture: 0)));
1657
1658 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: &Inst))
1659 return Builder.CreateBinOp(Opc: BinOp->getOpcode(),
1660 LHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 0)),
1661 RHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 1)));
1662
1663 if (isa<UIToFPInst>(Val: &Inst) || isa<SIToFPInst>(Val: &Inst)) {
1664 auto *Cast = cast<CastInst>(Val: &Inst);
1665 return Builder.CreateCast(Op: Cast->getOpcode(), V: Cast->getOperand(i_nocapture: 0),
1666 DestTy: ExtendedVT);
1667 }
1668
1669 if (auto *S = dyn_cast<SelectInst>(Val: &Inst))
1670 return Builder.CreateSelect(C: S->getCondition(),
1671 True: Map.getShadow(V: S->getTrueValue()),
1672 False: Map.getShadow(V: S->getFalseValue()));
1673
1674 if (auto *Freeze = dyn_cast<FreezeInst>(Val: &Inst))
1675 return Builder.CreateFreeze(V: Map.getShadow(V: Freeze->getOperand(i_nocapture: 0)));
1676
1677 if (auto *Extract = dyn_cast<ExtractElementInst>(Val: &Inst))
1678 return Builder.CreateExtractElement(
1679 Vec: Map.getShadow(V: Extract->getVectorOperand()), Idx: Extract->getIndexOperand());
1680
1681 if (auto *Insert = dyn_cast<InsertElementInst>(Val: &Inst))
1682 return Builder.CreateInsertElement(Vec: Map.getShadow(V: Insert->getOperand(i_nocapture: 0)),
1683 NewElt: Map.getShadow(V: Insert->getOperand(i_nocapture: 1)),
1684 Idx: Insert->getOperand(i_nocapture: 2));
1685
1686 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: &Inst))
1687 return Builder.CreateShuffleVector(V1: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 0)),
1688 V2: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 1)),
1689 Mask: Shuffle->getShuffleMask());
1690 // TODO: We could make aggregate object first class citizens. For now we
1691 // just extend the extracted value.
1692 if (auto *Extract = dyn_cast<ExtractValueInst>(Val: &Inst))
1693 return Builder.CreateFPExt(V: Extract, DestTy: ExtendedVT);
1694
1695 if (auto *BC = dyn_cast<BitCastInst>(Val: &Inst))
1696 return Builder.CreateFPExt(V: BC, DestTy: ExtendedVT);
1697
1698 report_fatal_error(reason: "Unimplemented support for " +
1699 Twine(Inst.getOpcodeName()));
1700}
1701
1702// Creates a shadow value for an instruction that defines a value of FT type.
1703// FT operands that do not already have shadow values are created recursively.
1704// The DFS is guaranteed to not loop as phis and arguments already have
1705// shadows.
1706void NumericalStabilitySanitizer::maybeCreateShadowValue(
1707 Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
1708 Type *VT = Root.getType();
1709 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1710 if (ExtendedVT == nullptr)
1711 return; // Not an FT value.
1712
1713 if (Map.hasShadow(V: &Root))
1714 return; // Shadow already exists.
1715
1716 assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");
1717
1718 std::vector<Instruction *> DfsStack(1, &Root);
1719 while (!DfsStack.empty()) {
1720 // Ensure that all operands to the instruction have shadows before
1721 // proceeding.
1722 Instruction *I = DfsStack.back();
1723 // The shadow for the instruction might have been created deeper in the DFS,
1724 // see `forward_use_with_two_uses` test.
1725 if (Map.hasShadow(V: I)) {
1726 DfsStack.pop_back();
1727 continue;
1728 }
1729
1730 bool MissingShadow = false;
1731 for (Value *Op : I->operands()) {
1732 Type *VT = Op->getType();
1733 if (!Config.getExtendedFPType(FT: VT))
1734 continue; // Not an FT value.
1735 if (Map.hasShadow(V: Op))
1736 continue; // Shadow is already available.
1737 MissingShadow = true;
1738 DfsStack.push_back(x: cast<Instruction>(Val: Op));
1739 }
1740 if (MissingShadow)
1741 continue; // Process operands and come back to this instruction later.
1742
1743 // All operands have shadows. Create a shadow for the current value.
1744 Value *Shadow = createShadowValueWithOperandsAvailable(Inst&: *I, TLI, Map);
1745 Map.setShadow(V&: *I, Shadow&: *Shadow);
1746 DfsStack.pop_back();
1747 }
1748}
1749
1750// A floating-point store needs its value and type written to shadow memory.
1751void NumericalStabilitySanitizer::propagateFTStore(
1752 StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {
1753 Value *StoredValue = Store.getValueOperand();
1754 IRBuilder<> Builder(&Store);
1755 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1756 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1757 Value *ShadowPtr = Builder.CreateCall(
1758 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1759 Args: {Store.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1760
1761 Value *StoredShadow = Map.getShadow(V: StoredValue);
1762 if (!Store.getParent()->getParent()->hasOptNone()) {
1763 // Only check stores when optimizing, because non-optimized code generates
1764 // too many stores to the stack, creating false positives.
1765 if (Opts.nsan_check_stores) {
1766 StoredShadow = emitCheck(V: StoredValue, ShadowV: StoredShadow, Builder,
1767 Loc: CheckLoc::makeStore(Address: Store.getPointerOperand()));
1768 ++NumInstrumentedFTStores;
1769 }
1770 }
1771
1772 Builder.CreateAlignedStore(Val: StoredShadow, Ptr: ShadowPtr, Align: Align(1),
1773 isVolatile: Store.isVolatile());
1774}
1775
1776// A non-ft store needs to invalidate shadow memory. Exceptions are:
1777// - memory transfers of floating-point data through other pointer types (llvm
1778// optimization passes transform `*(float*)a = *(float*)b` into
1779// `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.
1780// - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted
1781// ints. Note that this is not really necessary because if the value is
1782// unknown the framework will re-extend it on load anyway. It just felt
1783// easier to debug tests with vectors of FTs.
1784void NumericalStabilitySanitizer::propagateNonFTStore(
1785 StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {
1786 Value *PtrOp = Store.getPointerOperand();
1787 IRBuilder<> Builder(Store.getNextNode());
1788 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1789 Value *Dst = PtrOp;
1790 TypeSize SlotSize = DL.getTypeStoreSize(Ty: VT);
1791 assert(!SlotSize.isScalable() && "unsupported");
1792 const auto LoadSizeBytes = SlotSize.getFixedValue();
1793 Value *ValueSize = Constant::getIntegerValue(
1794 Ty: IntptrTy, V: APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));
1795
1796 ++NumInstrumentedNonFTStores;
1797 Value *StoredValue = Store.getValueOperand();
1798 if (LoadInst *Load = dyn_cast<LoadInst>(Val: StoredValue)) {
1799 // TODO: Handle the case when the value is from a phi.
1800 // This is a memory transfer with memcpy semantics. Copy the type and
1801 // value from the source. Note that we cannot use __nsan_copy_values()
1802 // here, because that will not work when there is a write to memory in
1803 // between the load and the store, e.g. in the case of a swap.
1804 Type *ShadowTypeIntTy = Type::getIntNTy(C&: Context, N: 8 * LoadSizeBytes);
1805 Type *ShadowValueIntTy =
1806 Type::getIntNTy(C&: Context, N: 8 * kShadowScale * LoadSizeBytes);
1807 IRBuilder<> LoadBuilder(Load->getNextNode());
1808 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1809 Value *LoadSrc = Load->getPointerOperand();
1810 // Read the shadow type and value at load time. The type has the same size
1811 // as the FT value, the value has twice its size.
1812 // TODO: cache them to avoid re-creating them when a load is used by
1813 // several stores. Maybe create them like the FT shadows when a load is
1814 // encountered.
1815 Value *RawShadowType = LoadBuilder.CreateAlignedLoad(
1816 Ty: ShadowTypeIntTy,
1817 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {LoadSrc}), Align: Align(1),
1818 /*isVolatile=*/false);
1819 Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(
1820 Ty: ShadowValueIntTy,
1821 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {LoadSrc}), Align: Align(1),
1822 /*isVolatile=*/false);
1823
1824 // Write back the shadow type and value at store time.
1825 Builder.CreateAlignedStore(
1826 Val: RawShadowType, Ptr: Builder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {Dst}),
1827 Align: Align(1),
1828 /*isVolatile=*/false);
1829 Builder.CreateAlignedStore(Val: RawShadowValue,
1830 Ptr: Builder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {Dst}),
1831 Align: Align(1),
1832 /*isVolatile=*/false);
1833
1834 ++NumInstrumentedNonFTMemcpyStores;
1835 return;
1836 }
1837 if (Constant *C; Opts.nsan_propagate_non_ft_const_stores_as_ft &&
1838 (C = dyn_cast<Constant>(Val: StoredValue))) {
1839 // This might be a fp constant stored as an int. Bitcast and store if it has
1840 // appropriate size.
1841 Type *BitcastTy = nullptr; // The FT type to bitcast to.
1842 if (isa<ConstantInt, ConstantDataVector>(Val: C)) {
1843 switch (C->getType()->getScalarSizeInBits()) {
1844 case 32:
1845 BitcastTy = Type::getFloatTy(C&: Context);
1846 break;
1847 case 64:
1848 BitcastTy = Type::getDoubleTy(C&: Context);
1849 break;
1850 case 80:
1851 BitcastTy = Type::getX86_FP80Ty(C&: Context);
1852 break;
1853 default:
1854 break;
1855 }
1856
1857 if (BitcastTy)
1858 if (auto *VectorTy = dyn_cast<VectorType>(Val: C->getType()))
1859 BitcastTy = VectorType::get(ElementType: BitcastTy, EC: VectorTy->getElementCount());
1860 }
1861 if (BitcastTy) {
1862 const MemoryExtents Extents = getMemoryExtentsOrDie(FT: BitcastTy);
1863 Value *ShadowPtr = Builder.CreateCall(
1864 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1865 Args: {PtrOp, ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1866 // Bitcast the integer value to the appropriate FT type and extend to 2FT.
1867 Type *ExtVT = Config.getExtendedFPType(FT: BitcastTy);
1868 Value *Shadow =
1869 Builder.CreateFPExt(V: Builder.CreateBitCast(V: C, DestTy: BitcastTy), DestTy: ExtVT);
1870 Builder.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1),
1871 isVolatile: Store.isVolatile());
1872 return;
1873 }
1874 }
1875 // All other stores just reset the shadow value to unknown.
1876 Builder.CreateCall(Callee: NsanSetUnknownFns.getFallback(), Args: {Dst, ValueSize});
1877}
1878
1879void NumericalStabilitySanitizer::propagateShadowValues(
1880 Instruction &Inst, const TargetLibraryInfo &TLI,
1881 const ValueToShadowMap &Map) {
1882 if (auto *Store = dyn_cast<StoreInst>(Val: &Inst)) {
1883 Value *StoredValue = Store->getValueOperand();
1884 Type *VT = StoredValue->getType();
1885 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1886 if (ExtendedVT == nullptr)
1887 return propagateNonFTStore(Store&: *Store, VT, Map);
1888 return propagateFTStore(Store&: *Store, VT, ExtendedVT, Map);
1889 }
1890
1891 if (auto *FCmp = dyn_cast<FCmpInst>(Val: &Inst)) {
1892 emitFCmpCheck(FCmp&: *FCmp, Map);
1893 return;
1894 }
1895
1896 if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
1897 maybeAddSuffixForNsanInterface(CI: CB);
1898 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
1899 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
1900 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
1901 instrumentMemIntrinsic(MI);
1902 return;
1903 }
1904 populateShadowStack(CI&: *CB, TLI, Map);
1905 return;
1906 }
1907
1908 if (auto *RetInst = dyn_cast<ReturnInst>(Val: &Inst)) {
1909 if (!Opts.nsan_check_ret)
1910 return;
1911
1912 Value *RV = RetInst->getReturnValue();
1913 if (RV == nullptr)
1914 return; // This is a `ret void`.
1915 Type *VT = RV->getType();
1916 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1917 if (ExtendedVT == nullptr)
1918 return; // Not an FT ret.
1919 Value *RVShadow = Map.getShadow(V: RV);
1920 IRBuilder<> Builder(RetInst);
1921
1922 RVShadow = emitCheck(V: RV, ShadowV: RVShadow, Builder, Loc: CheckLoc::makeRet());
1923 ++NumInstrumentedFTRets;
1924 // Store tag.
1925 Value *FnAddr =
1926 Builder.CreatePtrToInt(V: Inst.getParent()->getParent(), DestTy: IntptrTy);
1927 Builder.CreateStore(Val: FnAddr, Ptr: NsanShadowRetTag);
1928 // Store value.
1929 Value *ShadowRetValPtr =
1930 Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0);
1931 Builder.CreateStore(Val: RVShadow, Ptr: ShadowRetValPtr);
1932 return;
1933 }
1934
1935 if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(Val: &Inst)) {
1936 Value *V = Insert->getOperand(i_nocapture: 1);
1937 Type *VT = V->getType();
1938 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1939 if (ExtendedVT == nullptr)
1940 return;
1941 IRBuilder<> Builder(Insert);
1942 emitCheck(V, ShadowV: Map.getShadow(V), Builder, Loc: CheckLoc::makeInsert());
1943 return;
1944 }
1945}
1946
1947// Moves fast math flags from the function to individual instructions, and
1948// removes the attribute from the function.
1949// TODO: Make this controllable with a flag.
1950static void moveFastMathFlags(Function &F,
1951 std::vector<Instruction *> &Instructions) {
1952 FastMathFlags FMF;
1953#define MOVE_FLAG(attr, setter) \
1954 if (F.getFnAttribute(attr).getValueAsString() == "true") { \
1955 F.removeFnAttr(attr); \
1956 FMF.set##setter(); \
1957 }
1958 MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)
1959#undef MOVE_FLAG
1960
1961 for (Instruction *I : Instructions)
1962 if (isa<FPMathOperator>(Val: I))
1963 I->setFastMathFlags(FMF);
1964}
1965
1966bool NumericalStabilitySanitizer::sanitizeFunction(
1967 Function &F, const TargetLibraryInfo &TLI) {
1968 if (!F.hasFnAttribute(Kind: Attribute::SanitizeNumericalStability) ||
1969 F.isDeclaration())
1970 return false;
1971
1972 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
1973 return false;
1974
1975 // This is required to prevent instrumenting call to __nsan_init from within
1976 // the module constructor.
1977 if (F.getName() == kNsanModuleCtorName)
1978 return false;
1979
1980 // The instrumentation maintains:
1981 // - for each IR value `v` of floating-point (or vector floating-point) type
1982 // FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.
1983 // double for float and f128 for double).
1984 // - A shadow memory, which stores `s(v)` for any `v` that has been stored,
1985 // along with a shadow memory tag, which stores whether the value in the
1986 // corresponding shadow memory is valid. Note that this might be
1987 // incorrect if a non-instrumented function stores to memory, or if
1988 // memory is stored to through a char pointer.
1989 // - A shadow stack, which holds `s(v)` for any floating-point argument `v`
1990 // of a call to an instrumented function. This allows
1991 // instrumented functions to retrieve the shadow values for their
1992 // arguments.
1993 // Because instrumented functions can be called from non-instrumented
1994 // functions, the stack needs to include a tag so that the instrumented
1995 // function knows whether shadow values are available for their
1996 // parameters (i.e. whether is was called by an instrumented function).
1997 // When shadow arguments are not available, they have to be recreated by
1998 // extending the precision of the non-shadow arguments to the non-shadow
1999 // value. Non-instrumented functions do not modify (or even know about) the
2000 // shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow
2001 // stack tag is __nsan_shadow_args_tag. The tag is any unique identifier
2002 // for the function (we use the address of the function). Both variables
2003 // are thread local.
2004 // Example:
2005 // calls shadow stack tag shadow stack
2006 // =======================================================================
2007 // non_instrumented_1() 0 0
2008 // |
2009 // v
2010 // instrumented_2(float a) 0 0
2011 // |
2012 // v
2013 // instrumented_3(float b, double c) &instrumented_3 s(b),s(c)
2014 // |
2015 // v
2016 // instrumented_4(float d) &instrumented_4 s(d)
2017 // |
2018 // v
2019 // non_instrumented_5(float e) &non_instrumented_5 s(e)
2020 // |
2021 // v
2022 // instrumented_6(float f) &non_instrumented_5 s(e)
2023 //
2024 // On entry, instrumented_2 checks whether the tag corresponds to its
2025 // function ptr.
2026 // Note that functions reset the tag to 0 after reading shadow parameters.
2027 // This ensures that the function does not erroneously read invalid data if
2028 // called twice in the same stack, once from an instrumented function and
2029 // once from an uninstrumented one. For example, in the following example,
2030 // resetting the tag in (A) ensures that (B) does not reuse the same the
2031 // shadow arguments (which would be incorrect).
2032 // instrumented_1(float a)
2033 // |
2034 // v
2035 // instrumented_2(float b) (A)
2036 // |
2037 // v
2038 // non_instrumented_3()
2039 // |
2040 // v
2041 // instrumented_2(float b) (B)
2042 //
2043 // - A shadow return slot. Any function that returns a floating-point value
2044 // places a shadow return value in __nsan_shadow_ret_val. Again, because
2045 // we might be calling non-instrumented functions, this value is guarded
2046 // by __nsan_shadow_ret_tag marker indicating which instrumented function
2047 // placed the value in __nsan_shadow_ret_val, so that the caller can check
2048 // that this corresponds to the callee. Both variables are thread local.
2049 //
2050 // For example, in the following example, the instrumentation in
2051 // `instrumented_1` rejects the shadow return value from `instrumented_3`
2052 // because it is not tagged as expected (`&instrumented_3` instead of
2053 // `non_instrumented_2`):
2054 //
2055 // instrumented_1()
2056 // |
2057 // v
2058 // float non_instrumented_2()
2059 // |
2060 // v
2061 // float instrumented_3()
2062 //
2063 // Calls of known math functions (sin, cos, exp, ...) are duplicated to call
2064 // their overload on the shadow type.
2065
2066 // Collect all instructions before processing, as creating shadow values
2067 // creates new instructions inside the function.
2068 std::vector<Instruction *> OriginalInstructions;
2069 for (BasicBlock &BB : F)
2070 for (Instruction &Inst : BB)
2071 OriginalInstructions.emplace_back(args: &Inst);
2072
2073 moveFastMathFlags(F, Instructions&: OriginalInstructions);
2074 ValueToShadowMap ValueToShadow(Config);
2075
2076 // In the first pass, we create shadow values for all FT function arguments
2077 // and all phis. This ensures that the DFS of the next pass does not have
2078 // any loops.
2079 std::vector<PHINode *> OriginalPhis;
2080 createShadowArguments(F, TLI, Map&: ValueToShadow);
2081 for (Instruction *I : OriginalInstructions) {
2082 if (PHINode *Phi = dyn_cast<PHINode>(Val: I)) {
2083 if (PHINode *Shadow = maybeCreateShadowPhi(Phi&: *Phi, TLI)) {
2084 OriginalPhis.push_back(x: Phi);
2085 ValueToShadow.setShadow(V&: *Phi, Shadow&: *Shadow);
2086 }
2087 }
2088 }
2089
2090 // Create shadow values for all instructions creating FT values.
2091 for (Instruction *I : OriginalInstructions)
2092 maybeCreateShadowValue(Root&: *I, TLI, Map&: ValueToShadow);
2093
2094 // Propagate shadow values across stores, calls and rets.
2095 for (Instruction *I : OriginalInstructions)
2096 propagateShadowValues(Inst&: *I, TLI, Map: ValueToShadow);
2097
2098 // The last pass populates shadow phis with shadow values.
2099 for (PHINode *Phi : OriginalPhis) {
2100 PHINode *ShadowPhi = cast<PHINode>(Val: ValueToShadow.getShadow(V: Phi));
2101 for (unsigned I : seq(Size: Phi->getNumOperands())) {
2102 Value *V = Phi->getOperand(i_nocapture: I);
2103 Value *Shadow = ValueToShadow.getShadow(V);
2104 BasicBlock *IncomingBB = Phi->getIncomingBlock(i: I);
2105 // For some instructions (e.g. invoke), we create the shadow in a separate
2106 // block, different from the block where the original value is created.
2107 // In that case, the shadow phi might need to refer to this block instead
2108 // of the original block.
2109 // Note that this can only happen for instructions as constant shadows are
2110 // always created in the same block.
2111 ShadowPhi->addIncoming(V: Shadow, BB: IncomingBB);
2112 }
2113 }
2114
2115 return !ValueToShadow.empty();
2116}
2117
2118static uint64_t GetMemOpSize(Value *V) {
2119 uint64_t OpSize = 0;
2120 if (Constant *C = dyn_cast<Constant>(Val: V)) {
2121 auto *CInt = dyn_cast<ConstantInt>(Val: C);
2122 if (CInt && CInt->getValue().getBitWidth() <= 64)
2123 OpSize = CInt->getValue().getZExtValue();
2124 }
2125
2126 return OpSize;
2127}
2128
2129// Instrument the memory intrinsics so that they properly modify the shadow
2130// memory.
2131bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
2132 IRBuilder<> Builder(MI);
2133 if (auto *M = dyn_cast<MemSetInst>(Val: MI)) {
2134 FunctionCallee SetUnknownFn =
2135 NsanSetUnknownFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2136 if (SetUnknownFn.getFunctionType()->getNumParams() == 1)
2137 Builder.CreateCall(Callee: SetUnknownFn, Args: {/*Address=*/M->getArgOperand(i: 0)});
2138 else
2139 Builder.CreateCall(Callee: SetUnknownFn,
2140 Args: {/*Address=*/M->getArgOperand(i: 0),
2141 /*Size=*/Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2142 DestTy: IntptrTy, isSigned: false)});
2143
2144 } else if (auto *M = dyn_cast<MemTransferInst>(Val: MI)) {
2145 FunctionCallee CopyFn =
2146 NsanCopyFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2147
2148 if (CopyFn.getFunctionType()->getNumParams() == 2)
2149 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2150 /*Source=*/M->getArgOperand(i: 1)});
2151 else
2152 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2153 /*Source=*/M->getArgOperand(i: 1),
2154 /*Size=*/
2155 Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2156 DestTy: IntptrTy, isSigned: false)});
2157 }
2158 return false;
2159}
2160
2161void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {
2162 Function *Fn = CI->getCalledFunction();
2163 if (Fn == nullptr)
2164 return;
2165
2166 if (!Fn->getName().starts_with(Prefix: "__nsan_"))
2167 return;
2168
2169 if (Fn->getName() == "__nsan_dump_shadow_mem") {
2170 assert(CI->arg_size() == 4 &&
2171 "invalid prototype for __nsan_dump_shadow_mem");
2172 // __nsan_dump_shadow_mem requires an extra parameter with the dynamic
2173 // configuration:
2174 // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)
2175 // | shadow_type_id_for_double
2176 const uint64_t shadow_value_type_ids =
2177 (static_cast<size_t>(Config.byValueType(VT: kLongDouble).getNsanTypeId())
2178 << 16) |
2179 (static_cast<size_t>(Config.byValueType(VT: kDouble).getNsanTypeId())
2180 << 8) |
2181 static_cast<size_t>(Config.byValueType(VT: kFloat).getNsanTypeId());
2182 CI->setArgOperand(i: 3, v: ConstantInt::get(Ty: IntptrTy, V: shadow_value_type_ids));
2183 }
2184}
2185