1//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the instrumentation pass for the numerical sanitizer.
10// Conceptually the pass injects shadow computations using higher precision
11// types and inserts consistency checks. For details see the paper
12// https://arxiv.org/abs/2102.12782.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
17#include "InstrumentationOptions.h"
18
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/IntrinsicInst.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/Type.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/Regex.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Transforms/Utils/BasicBlockUtils.h"
39#include "llvm/Transforms/Utils/Instrumentation.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43#include <cstdint>
44
45using namespace llvm;
46
47#define DEBUG_TYPE "nsan"
48
49STATISTIC(NumInstrumentedFTLoads,
50 "Number of instrumented floating-point loads");
51
52STATISTIC(NumInstrumentedFTCalls,
53 "Number of instrumented floating-point calls");
54STATISTIC(NumInstrumentedFTRets,
55 "Number of instrumented floating-point returns");
56STATISTIC(NumInstrumentedFTStores,
57 "Number of instrumented floating-point stores");
58STATISTIC(NumInstrumentedNonFTStores,
59 "Number of instrumented non floating-point stores");
60STATISTIC(
61 NumInstrumentedNonFTMemcpyStores,
62 "Number of instrumented non floating-point stores with memcpy semantics");
63STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
64
65constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
66constexpr StringLiteral kNsanInitName("__nsan_init");
67
68// The following values must be kept in sync with the runtime.
69constexpr int kShadowScale = 2;
70constexpr int kMaxVectorWidth = 8;
71constexpr int kMaxNumArgs = 128;
72constexpr int kMaxShadowTypeSizeBytes = 16; // fp128
73
74namespace {
75
76// Defines the characteristics (type id, type, and floating-point semantics)
77// attached for all possible shadow types.
78class ShadowTypeConfig {
79public:
80 static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
81
82 // The LLVM Type corresponding to the shadow type.
83 virtual Type *getType(LLVMContext &Context) const = 0;
84
85 // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
86 virtual char getNsanTypeId() const = 0;
87
88 virtual ~ShadowTypeConfig() = default;
89};
90
91template <char NsanTypeId>
92class ShadowTypeConfigImpl : public ShadowTypeConfig {
93public:
94 char getNsanTypeId() const override { return NsanTypeId; }
95 static constexpr char kNsanTypeId = NsanTypeId;
96};
97
98// `double` (`d`) shadow type.
99class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
100 Type *getType(LLVMContext &Context) const override {
101 return Type::getDoubleTy(C&: Context);
102 }
103};
104
105// `x86_fp80` (`l`) shadow type: X86 long double.
106class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
107 Type *getType(LLVMContext &Context) const override {
108 return Type::getX86_FP80Ty(C&: Context);
109 }
110};
111
112// `fp128` (`q`) shadow type.
113class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
114 Type *getType(LLVMContext &Context) const override {
115 return Type::getFP128Ty(C&: Context);
116 }
117};
118
119// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
120class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
121 Type *getType(LLVMContext &Context) const override {
122 return Type::getPPC_FP128Ty(C&: Context);
123 }
124};
125
126// Creates a ShadowTypeConfig given its type id.
127std::unique_ptr<ShadowTypeConfig>
128ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
129 switch (TypeId) {
130 case F64ShadowConfig::kNsanTypeId:
131 return std::make_unique<F64ShadowConfig>();
132 case F80ShadowConfig::kNsanTypeId:
133 return std::make_unique<F80ShadowConfig>();
134 case F128ShadowConfig::kNsanTypeId:
135 return std::make_unique<F128ShadowConfig>();
136 case PPC128ShadowConfig::kNsanTypeId:
137 return std::make_unique<PPC128ShadowConfig>();
138 }
139 report_fatal_error(reason: "nsan: invalid shadow type id '" + Twine(TypeId) + "'");
140}
141
142// An enum corresponding to shadow value types. Used as indices in arrays, so
143// not an `enum class`.
144enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
145
146// If `FT` corresponds to a primitive FTValueType, return it.
147static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
148 if (FT->isFloatTy())
149 return kFloat;
150 if (FT->isDoubleTy())
151 return kDouble;
152 if (FT->isX86_FP80Ty())
153 return kLongDouble;
154 return {};
155}
156
157// Returns the LLVM type for an FTValueType.
158static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
159 switch (VT) {
160 case kFloat:
161 return Type::getFloatTy(C&: Context);
162 case kDouble:
163 return Type::getDoubleTy(C&: Context);
164 case kLongDouble:
165 return Type::getX86_FP80Ty(C&: Context);
166 case kNumValueTypes:
167 return nullptr;
168 }
169 llvm_unreachable("Unhandled FTValueType enum");
170}
171
172// Returns the type name for an FTValueType.
173static const char *typeNameFromFTValueType(FTValueType VT) {
174 switch (VT) {
175 case kFloat:
176 return "float";
177 case kDouble:
178 return "double";
179 case kLongDouble:
180 return "longdouble";
181 case kNumValueTypes:
182 return nullptr;
183 }
184 llvm_unreachable("Unhandled FTValueType enum");
185}
186
187// A specific mapping configuration of application type to shadow type for nsan
188// (see -nsan-shadow-mapping flag).
189class MappingConfig {
190public:
191 MappingConfig(const InstrumentationOptions &Opts, LLVMContext &C)
192 : Context(C) {
193 if (Opts.nsan_shadow_type_mapping.size() != 3)
194 report_fatal_error(reason: "Invalid nsan mapping: " +
195 Twine(Opts.nsan_shadow_type_mapping));
196 unsigned ShadowTypeSizeBits[kNumValueTypes];
197 for (int VT = 0; VT < kNumValueTypes; ++VT) {
198 auto Config =
199 ShadowTypeConfig::fromNsanTypeId(TypeId: Opts.nsan_shadow_type_mapping[VT]);
200 if (!Config)
201 report_fatal_error(reason: "Failed to get ShadowTypeConfig for " +
202 Twine(Opts.nsan_shadow_type_mapping[VT]));
203 const unsigned AppTypeSize =
204 typeFromFTValueType(VT: static_cast<FTValueType>(VT), Context)
205 ->getScalarSizeInBits();
206 const unsigned ShadowTypeSize =
207 Config->getType(Context)->getScalarSizeInBits();
208 // Check that the shadow type size is at most kShadowScale times the
209 // application type size, so that shadow memory compoutations are valid.
210 if (ShadowTypeSize > kShadowScale * AppTypeSize)
211 report_fatal_error(reason: "Invalid nsan mapping f" + Twine(AppTypeSize) +
212 "->f" + Twine(ShadowTypeSize) +
213 ": The shadow type size should be at most " +
214 Twine(kShadowScale) +
215 " times the application type size");
216 ShadowTypeSizeBits[VT] = ShadowTypeSize;
217 Configs[VT] = std::move(Config);
218 }
219
220 // Check that the mapping is monotonous. This is required because if one
221 // does an fpextend of `float->long double` in application code, nsan is
222 // going to do an fpextend of `shadow(float) -> shadow(long double)` in
223 // shadow code. This will fail in `qql` mode, since nsan would be
224 // fpextending `f128->long`, which is invalid.
225 // TODO: Relax this.
226 if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
227 ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
228 report_fatal_error(reason: "Invalid nsan mapping: { float->f" +
229 Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
230 Twine(ShadowTypeSizeBits[kDouble]) +
231 "; long double->f" +
232 Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
233 }
234
235 const ShadowTypeConfig &byValueType(FTValueType VT) const {
236 assert(VT < FTValueType::kNumValueTypes && "invalid value type");
237 return *Configs[VT];
238 }
239
240 // Returns the extended shadow type for a given application type.
241 Type *getExtendedFPType(Type *FT) const {
242 if (const auto VT = ftValueTypeFromType(FT))
243 return Configs[*VT]->getType(Context);
244 if (FT->isVectorTy()) {
245 auto *VecTy = cast<VectorType>(Val: FT);
246 // TODO: add support for scalable vector types.
247 if (VecTy->isScalableTy())
248 return nullptr;
249 Type *ExtendedScalar = getExtendedFPType(FT: VecTy->getElementType());
250 return ExtendedScalar
251 ? VectorType::get(ElementType: ExtendedScalar, EC: VecTy->getElementCount())
252 : nullptr;
253 }
254 return nullptr;
255 }
256
257private:
258 LLVMContext &Context;
259 std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
260};
261
262// The memory extents of a type specifies how many elements of a given
263// FTValueType needs to be stored when storing this type.
264struct MemoryExtents {
265 FTValueType ValueType;
266 uint64_t NumElts;
267};
268
269static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
270 if (const auto VT = ftValueTypeFromType(FT))
271 return {.ValueType: *VT, .NumElts: 1};
272 if (auto *VecTy = dyn_cast<VectorType>(Val: FT)) {
273 const auto ScalarExtents = getMemoryExtentsOrDie(FT: VecTy->getElementType());
274 return {.ValueType: ScalarExtents.ValueType,
275 .NumElts: ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
276 }
277 llvm_unreachable("invalid value type");
278}
279
280// The location of a check. Passed as parameters to runtime checking functions.
281class CheckLoc {
282public:
283 // Creates a location that references an application memory location.
284 static CheckLoc makeStore(Value *Address) {
285 CheckLoc Result(kStore);
286 Result.Address = Address;
287 return Result;
288 }
289 static CheckLoc makeLoad(Value *Address) {
290 CheckLoc Result(kLoad);
291 Result.Address = Address;
292 return Result;
293 }
294
295 // Creates a location that references an argument, given by id.
296 static CheckLoc makeArg(int ArgId) {
297 CheckLoc Result(kArg);
298 Result.ArgId = ArgId;
299 return Result;
300 }
301
302 // Creates a location that references the return value of a function.
303 static CheckLoc makeRet() { return CheckLoc(kRet); }
304
305 // Creates a location that references a vector insert.
306 static CheckLoc makeInsert() { return CheckLoc(kInsert); }
307
308 // Returns the CheckType of location this refers to, as an integer-typed LLVM
309 // IR value.
310 Value *getType(LLVMContext &C) const {
311 return ConstantInt::get(Ty: Type::getInt32Ty(C), V: static_cast<int>(CheckTy));
312 }
313
314 // Returns a CheckType-specific value representing details of the location
315 // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
316 // IR value.
317 Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
318 switch (CheckTy) {
319 case kUnknown:
320 llvm_unreachable("unknown type");
321 case kRet:
322 case kInsert:
323 return ConstantInt::get(Ty: IntptrTy, V: 0);
324 case kArg:
325 return ConstantInt::get(Ty: IntptrTy, V: ArgId);
326 case kLoad:
327 case kStore:
328 return Builder.CreatePtrToInt(V: Address, DestTy: IntptrTy);
329 }
330 llvm_unreachable("Unhandled CheckType enum");
331 }
332
333private:
334 // Must be kept in sync with the runtime,
335 // see compiler-rt/lib/nsan/nsan_stats.h
336 enum CheckType {
337 kUnknown = 0,
338 kRet,
339 kArg,
340 kLoad,
341 kStore,
342 kInsert,
343 };
344 explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
345
346 Value *Address = nullptr;
347 const CheckType CheckTy;
348 int ArgId = -1;
349};
350
351// A map of LLVM IR values to shadow LLVM IR values.
352class ValueToShadowMap {
353public:
354 explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}
355
356 ValueToShadowMap(const ValueToShadowMap &) = delete;
357 ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;
358
359 // Sets the shadow value for a value. Asserts that the value does not already
360 // have a value.
361 void setShadow(Value &V, Value &Shadow) {
362 [[maybe_unused]] const bool Inserted = Map.try_emplace(Key: &V, Args: &Shadow).second;
363 LLVM_DEBUG({
364 if (!Inserted) {
365 if (auto *I = dyn_cast<Instruction>(&V))
366 errs() << I->getFunction()->getName() << ": ";
367 errs() << "duplicate shadow (" << &V << "): ";
368 V.dump();
369 }
370 });
371 assert(Inserted && "duplicate shadow");
372 }
373
374 // Returns true if the value already has a shadow (including if the value is a
375 // constant). If true, calling getShadow() is valid.
376 bool hasShadow(Value *V) const { return isa<Constant>(Val: V) || Map.contains(Val: V); }
377
378 // Returns the shadow value for a given value. Asserts that the value has
379 // a shadow value. Lazily creates shadows for constant values.
380 Value *getShadow(Value *V) const {
381 if (Constant *C = dyn_cast<Constant>(Val: V))
382 return getShadowConstant(C);
383 return Map.find(Val: V)->second;
384 }
385
386 bool empty() const { return Map.empty(); }
387
388private:
389 // Extends a constant application value to its shadow counterpart.
390 APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
391 bool LosesInfo = false;
392 CV.convert(ToSemantics: To, RM: APFloatBase::rmTowardZero, losesInfo: &LosesInfo);
393 return CV;
394 }
395
396 // Returns the shadow constant for the given application constant.
397 Constant *getShadowConstant(Constant *C) const {
398 if (UndefValue *U = dyn_cast<UndefValue>(Val: C)) {
399 return UndefValue::get(T: Config.getExtendedFPType(FT: U->getType()));
400 }
401 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
402 // Floating-point constants.
403 Type *Ty = Config.getExtendedFPType(FT: CFP->getType());
404 return ConstantFP::get(
405 Ty, V: extendConstantFP(CV: CFP->getValueAPF(),
406 To: Ty->getScalarType()->getFltSemantics()));
407 }
408 // Vector, array, or aggregate constants.
409 if (C->getType()->isVectorTy()) {
410 SmallVector<Constant *, 8> Elements;
411 for (int I = 0, E = cast<VectorType>(Val: C->getType())
412 ->getElementCount()
413 .getFixedValue();
414 I < E; ++I)
415 Elements.push_back(Elt: getShadowConstant(C: C->getAggregateElement(Elt: I)));
416 return ConstantVector::get(V: Elements);
417 }
418 llvm_unreachable("unimplemented");
419 }
420
421 const MappingConfig &Config;
422 DenseMap<Value *, Value *> Map;
423};
424
425class NsanMemOpFn {
426public:
427 NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,
428 size_t NumArgs);
429 FunctionCallee getFunctionFor(uint64_t MemOpSize) const;
430 FunctionCallee getFallback() const;
431
432private:
433 SmallVector<FunctionCallee> Funcs;
434 size_t NumSizedFuncs;
435};
436
437NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,
438 StringRef Fallback, size_t NumArgs) {
439 LLVMContext &Ctx = M.getContext();
440 AttributeList Attr;
441 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
442 Type *PtrTy = PointerType::getUnqual(C&: Ctx);
443 Type *VoidTy = Type::getVoidTy(C&: Ctx);
444 IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx);
445 FunctionType *SizedFnTy = nullptr;
446
447 NumSizedFuncs = Sized.size();
448
449 // First entry is fallback function
450 if (NumArgs == 3) {
451 Funcs.push_back(
452 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy));
453 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy, PtrTy}, isVarArg: false);
454 } else if (NumArgs == 2) {
455 Funcs.push_back(
456 Elt: M.getOrInsertFunction(Name: Fallback, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: IntptrTy));
457 SizedFnTy = FunctionType::get(Result: VoidTy, Params: {PtrTy}, isVarArg: false);
458 } else {
459 llvm_unreachable("Unexpected value of sized functions arguments");
460 }
461
462 for (size_t i = 0; i < NumSizedFuncs; ++i)
463 Funcs.push_back(Elt: M.getOrInsertFunction(Name: Sized[i], T: SizedFnTy, AttributeList: Attr));
464}
465
466FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {
467 // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the
468 // following code assumes that the number of functions in `Func` is sufficient
469 assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");
470
471 size_t Idx =
472 MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));
473
474 return Funcs[Idx];
475}
476
477FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }
478
479/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
480/// API function declarations into the module if they don't exist already.
481/// Instantiating ensures the __nsan_init function is in the list of global
482/// constructors for the module.
483class NumericalStabilitySanitizer {
484public:
485 NumericalStabilitySanitizer(const InstrumentationOptions &Opts, Module &M);
486 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
487
488private:
489 bool instrumentMemIntrinsic(MemIntrinsic *MI);
490 void maybeAddSuffixForNsanInterface(CallBase *CI);
491 bool addrPointsToConstantData(Value *Addr);
492 void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
493 ValueToShadowMap &Map);
494 Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
495 const TargetLibraryInfo &TLI,
496 const ValueToShadowMap &Map);
497 PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
498 void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
499 ValueToShadowMap &Map);
500
501 void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
502 const ValueToShadowMap &Map);
503
504 void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
505 const ValueToShadowMap &Map);
506 Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
507 CheckLoc Loc);
508 Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
509 CheckLoc Loc);
510 void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
511
512 // Value creation handlers.
513 Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
514 Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
515 const TargetLibraryInfo &TLI,
516 const ValueToShadowMap &Map, IRBuilder<> &Builder);
517 Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
518 const TargetLibraryInfo &TLI,
519 const ValueToShadowMap &Map,
520 IRBuilder<> &Builder);
521 Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
522 const ValueToShadowMap &Map, IRBuilder<> &Builder);
523 Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,
524 const ValueToShadowMap &Map, IRBuilder<> &Builder);
525
526 // Value propagation handlers.
527 void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
528 const ValueToShadowMap &Map);
529 void propagateNonFTStore(StoreInst &Store, Type *VT,
530 const ValueToShadowMap &Map);
531
532 const InstrumentationOptions &Opts;
533 const DataLayout &DL;
534 LLVMContext &Context;
535 MappingConfig Config;
536 IntegerType *IntptrTy = nullptr;
537
538 // TODO: Use std::array instead?
539 FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};
540 FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};
541 FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};
542 FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};
543
544 NsanMemOpFn NsanCopyFns;
545 NsanMemOpFn NsanSetUnknownFns;
546
547 FunctionCallee NsanGetRawShadowTypePtr;
548 FunctionCallee NsanGetRawShadowPtr;
549 GlobalValue *NsanShadowRetTag = nullptr;
550
551 Type *NsanShadowRetType = nullptr;
552 GlobalValue *NsanShadowRetPtr = nullptr;
553
554 GlobalValue *NsanShadowArgsTag = nullptr;
555
556 Type *NsanShadowArgsType = nullptr;
557 GlobalValue *NsanShadowArgsPtr = nullptr;
558
559 std::optional<Regex> CheckFunctionsFilter;
560};
561} // end anonymous namespace
562
563PreservedAnalyses
564NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
565 getOrCreateSanitizerCtorAndInitFunctions(
566 M, CtorName: kNsanModuleCtorName, InitName: kNsanInitName, /*InitArgTypes=*/{},
567 /*InitArgs=*/{},
568 // This callback is invoked when the functions are created the first
569 // time. Hook them into the global ctors list in that case:
570 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
571
572 const InstrumentationOptions &Opts =
573 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
574 NumericalStabilitySanitizer Nsan(Opts, M);
575 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
576 for (Function &F : M)
577 Nsan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
578
579 return PreservedAnalyses::none();
580}
581
582static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
583 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&M, Ty, Name] {
584 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
585 nullptr, Name, nullptr,
586 GlobalVariable::InitialExecTLSModel);
587 });
588}
589
590NumericalStabilitySanitizer::NumericalStabilitySanitizer(
591 const InstrumentationOptions &Opts, Module &M)
592 : Opts(Opts), DL(M.getDataLayout()), Context(M.getContext()),
593 Config(Opts, Context),
594 NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},
595 "__nsan_copy_values", /*NumArgs=*/3),
596 NsanSetUnknownFns(M,
597 {"__nsan_set_value_unknown_4",
598 "__nsan_set_value_unknown_8",
599 "__nsan_set_value_unknown_16"},
600 "__nsan_set_value_unknown", /*NumArgs=*/2) {
601 IntptrTy = DL.getIntPtrType(C&: Context);
602 Type *PtrTy = PointerType::getUnqual(C&: Context);
603 Type *Int32Ty = Type::getInt32Ty(C&: Context);
604 Type *Int1Ty = Type::getInt1Ty(C&: Context);
605 Type *VoidTy = Type::getVoidTy(C&: Context);
606
607 AttributeList Attr;
608 Attr = Attr.addFnAttribute(C&: Context, Kind: Attribute::NoUnwind);
609
610 Triple TT(M.getTargetTriple());
611 Attribute::AttrKind I32RetExtAttr =
612 TargetLibraryInfo::getExtAttrForI32Return(T: TT, /*Signed=*/true);
613 Attribute::AttrKind I32ArgExtAttr =
614 TargetLibraryInfo::getExtAttrForI32Param(T: TT, /*Signed=*/true);
615 Attribute::AttrKind BoolExtAttr =
616 TargetLibraryInfo::getExtAttrForBoolParam(T: TT);
617
618 // Initialize the runtime values (functions and global variables).
619 for (int I = 0; I < kNumValueTypes; ++I) {
620 const FTValueType VT = static_cast<FTValueType>(I);
621 const char *VTName = typeNameFromFTValueType(VT);
622 Type *VTTy = typeFromFTValueType(VT, Context);
623
624 // Load/store.
625 const std::string GetterPrefix =
626 std::string("__nsan_get_shadow_ptr_for_") + VTName;
627 NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
628 Name: GetterPrefix + "_store", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
629 NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
630 Name: GetterPrefix + "_load", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy, Args: IntptrTy);
631
632 // Check.
633 const auto &ShadowConfig = Config.byValueType(VT);
634 Type *ShadowTy = ShadowConfig.getType(Context);
635 {
636 AttributeList AL = Attr;
637 AL = AL.maybeAddRetAttribute(C&: Context, Kind: I32RetExtAttr);
638 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 2, Kind: I32ArgExtAttr);
639 NsanCheckValue[VT] =
640 M.getOrInsertFunction(Name: std::string("__nsan_internal_check_") + VTName +
641 "_" + ShadowConfig.getNsanTypeId(),
642 AttributeList: AL, RetTy: Int32Ty, Args: VTTy, Args: ShadowTy, Args: Int32Ty, Args: IntptrTy);
643 }
644
645 {
646 AttributeList AL = Attr;
647 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 4, Kind: I32ArgExtAttr);
648 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 5, Kind: BoolExtAttr);
649 AL = AL.maybeAddParamAttribute(C&: Context, ArgNo: 6, Kind: BoolExtAttr);
650 NsanFCmpFail[VT] = M.getOrInsertFunction(
651 Name: std::string("__nsan_fcmp_fail_") + VTName + "_" +
652 ShadowConfig.getNsanTypeId(),
653 AttributeList: AL, RetTy: VoidTy, Args: VTTy, Args: VTTy, Args: ShadowTy, Args: ShadowTy, Args: Int32Ty, Args: Int1Ty, Args: Int1Ty);
654 }
655 }
656
657 // TODO: Add attributes nofree, nosync, readnone, readonly,
658 NsanGetRawShadowTypePtr = M.getOrInsertFunction(
659 Name: "__nsan_internal_get_raw_shadow_type_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
660 NsanGetRawShadowPtr = M.getOrInsertFunction(
661 Name: "__nsan_internal_get_raw_shadow_ptr", AttributeList: Attr, RetTy: PtrTy, Args: PtrTy);
662
663 NsanShadowRetTag = createThreadLocalGV(Name: "__nsan_shadow_ret_tag", M, Ty: IntptrTy);
664
665 NsanShadowRetType = ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
666 NumElements: kMaxVectorWidth * kMaxShadowTypeSizeBytes);
667 NsanShadowRetPtr =
668 createThreadLocalGV(Name: "__nsan_shadow_ret_ptr", M, Ty: NsanShadowRetType);
669
670 NsanShadowArgsTag =
671 createThreadLocalGV(Name: "__nsan_shadow_args_tag", M, Ty: IntptrTy);
672
673 NsanShadowArgsType =
674 ArrayType::get(ElementType: Type::getInt8Ty(C&: Context),
675 NumElements: kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
676
677 NsanShadowArgsPtr =
678 createThreadLocalGV(Name: "__nsan_shadow_args_ptr", M, Ty: NsanShadowArgsType);
679
680 if (!Opts.check_functions_filter.empty()) {
681 Regex R = Regex(Opts.check_functions_filter);
682 std::string RegexError;
683 assert(R.isValid(RegexError));
684 CheckFunctionsFilter = std::move(R);
685 }
686}
687
688// Returns true if the given LLVM Value points to constant data (typically, a
689// global variable reference).
690bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
691 // If this is a GEP, just analyze its pointer operand.
692 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Addr))
693 Addr = GEP->getPointerOperand();
694
695 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr))
696 return GV->isConstant();
697 return false;
698}
699
700// This instruments the function entry to create shadow arguments.
701// Pseudocode:
702// if (this_fn_ptr == __nsan_shadow_args_tag) {
703// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
704// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
705// ...
706// __nsan_shadow_args_tag = 0;
707// } else {
708// s(arg0) = fext(arg0);
709// s(arg1) = fext(arg1);
710// ...
711// }
712void NumericalStabilitySanitizer::createShadowArguments(
713 Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
714 assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
715
716 // Do not bother if there are no FP args.
717 if (all_of(Range: F.args(), P: [this](const Argument &Arg) {
718 return Config.getExtendedFPType(FT: Arg.getType()) == nullptr;
719 }))
720 return;
721
722 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIIt());
723 // The function has shadow args if the shadow args tag matches the function
724 // address.
725 Value *HasShadowArgs = Builder.CreateICmpEQ(
726 LHS: Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowArgsTag, /*isVolatile=*/false),
727 RHS: Builder.CreatePtrToInt(V: &F, DestTy: IntptrTy));
728
729 unsigned ShadowArgsOffsetBytes = 0;
730 for (Argument &Arg : F.args()) {
731 Type *VT = Arg.getType();
732 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
733 if (ExtendedVT == nullptr)
734 continue; // Not an FT value.
735 Value *L = Builder.CreateAlignedLoad(
736 Ty: ExtendedVT,
737 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
738 Idx1: ShadowArgsOffsetBytes),
739 Align: Align(1), /*isVolatile=*/false);
740 Value *Shadow = Builder.CreateSelect(C: HasShadowArgs, True: L,
741 False: Builder.CreateFPExt(V: &Arg, DestTy: ExtendedVT));
742 Map.setShadow(V&: Arg, Shadow&: *Shadow);
743 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
744 assert(!SlotSize.isScalable() && "unsupported");
745 ShadowArgsOffsetBytes += SlotSize;
746 }
747 Builder.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: 0), Ptr: NsanShadowArgsTag);
748}
749
750// Returns true if the instrumentation should emit code to check arguments
751// before a function call.
752static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
753 const std::optional<Regex> &CheckFunctionsFilter) {
754
755 Function *Fn = CI.getCalledFunction();
756
757 if (CheckFunctionsFilter) {
758 // Skip checking args of indirect calls.
759 if (Fn == nullptr)
760 return false;
761 if (CheckFunctionsFilter->match(String: Fn->getName()))
762 return true;
763 return false;
764 }
765
766 if (Fn == nullptr)
767 return true; // Always check args of indirect calls.
768
769 // Never check nsan functions, the user called them for a reason.
770 if (Fn->getName().starts_with(Prefix: "__nsan_"))
771 return false;
772
773 const auto ID = Fn->getIntrinsicID();
774 LibFunc LFunc = TLI.getLibFunc(FDecl: *Fn);
775 // Always check args of unknown functions.
776 if (ID == Intrinsic::ID() && LFunc == NotLibFunc)
777 return true;
778
779 // Do not check args of an `fabs` call that is used for a comparison.
780 // This is typically used for `fabs(a-b) < tolerance`, where what matters is
781 // the result of the comparison, which is already caught be the fcmp checks.
782 if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
783 LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
784 for (const auto &U : CI.users())
785 if (isa<CmpInst>(Val: U))
786 return false;
787
788 return true; // Default is check.
789}
790
791// Populates the shadow call stack (which contains shadow values for every
792// floating-point parameter to the function).
793void NumericalStabilitySanitizer::populateShadowStack(
794 CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
795 // Do not create a shadow stack for inline asm.
796 if (CI.isInlineAsm())
797 return;
798
799 // Do not bother if there are no FP args.
800 if (all_of(Range: CI.operands(), P: [this](const Value *Arg) {
801 return Config.getExtendedFPType(FT: Arg->getType()) == nullptr;
802 }))
803 return;
804
805 IRBuilder<> Builder(&CI);
806 SmallVector<Value *, 8> ArgShadows;
807 const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
808 for (auto [ArgIdx, Arg] : enumerate(First: CI.operands())) {
809 if (Config.getExtendedFPType(FT: Arg->getType()) == nullptr)
810 continue; // Not an FT value.
811 Value *ArgShadow = Map.getShadow(V: Arg);
812 ArgShadows.push_back(Elt: ShouldCheckArgs ? emitCheck(V: Arg, ShadowV: ArgShadow, Builder,
813 Loc: CheckLoc::makeArg(ArgId: ArgIdx))
814 : ArgShadow);
815 }
816
817 // Do not create shadow stacks for intrinsics/known lib funcs.
818 if (Function *Fn = CI.getCalledFunction()) {
819 if (Fn->isIntrinsic() || TLI.getLibFunc(FDecl: *Fn) != NotLibFunc)
820 return;
821 }
822
823 // Set the shadow stack tag.
824 Builder.CreateStore(Val: CI.getCalledOperand(), Ptr: NsanShadowArgsTag);
825 TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(ExactSize: 0);
826
827 unsigned ShadowArgId = 0;
828 for (const Value *Arg : CI.operands()) {
829 Type *VT = Arg->getType();
830 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
831 if (ExtendedVT == nullptr)
832 continue; // Not an FT value.
833 Builder.CreateAlignedStore(
834 Val: ArgShadows[ShadowArgId++],
835 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowArgsType, Ptr: NsanShadowArgsPtr, Idx0: 0,
836 Idx1: ShadowArgsOffsetBytes),
837 Align: Align(1), /*isVolatile=*/false);
838 TypeSize SlotSize = DL.getTypeStoreSize(Ty: ExtendedVT);
839 assert(!SlotSize.isScalable() && "unsupported");
840 ShadowArgsOffsetBytes += SlotSize;
841 }
842}
843
844// Internal part of emitCheck(). Returns a value that indicates whether
845// computation should continue with the shadow or resume by re-fextending the
846// value.
847enum class ContinuationType { // Keep in sync with runtime.
848 ContinueWithShadow = 0,
849 ResumeFromValue = 1,
850};
851
852Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
853 IRBuilder<> &Builder,
854 CheckLoc Loc) {
855 // Do not emit checks for constant values, this is redundant.
856 if (isa<Constant>(Val: V))
857 return ConstantInt::get(
858 Ty: Builder.getInt32Ty(),
859 V: static_cast<int>(ContinuationType::ContinueWithShadow));
860
861 Type *Ty = V->getType();
862 if (const auto VT = ftValueTypeFromType(FT: Ty))
863 return Builder.CreateCall(
864 Callee: NsanCheckValue[*VT],
865 Args: {V, ShadowV, Loc.getType(C&: Context), Loc.getValue(IntptrTy, Builder)});
866
867 if (Ty->isVectorTy()) {
868 auto *VecTy = cast<VectorType>(Val: Ty);
869 // We currently skip scalable vector types in MappingConfig,
870 // thus we should not encounter any such types here.
871 assert(!VecTy->isScalableTy() &&
872 "Scalable vector types are not supported yet");
873 Value *CheckResult = nullptr;
874 for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
875 // We resume if any element resumes. Another option would be to create a
876 // vector shuffle with the array of ContinueWithShadow, but that is too
877 // complex.
878 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
879 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
880 Value *ComponentCheckResult =
881 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
882 CheckResult = CheckResult
883 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
884 : ComponentCheckResult;
885 }
886 return CheckResult;
887 }
888 if (Ty->isArrayTy()) {
889 Value *CheckResult = nullptr;
890 for (auto I : seq(Size: Ty->getArrayNumElements())) {
891 Value *ExtractV = Builder.CreateExtractElement(Vec: V, Idx: I);
892 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
893 Value *ComponentCheckResult =
894 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
895 CheckResult = CheckResult
896 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
897 : ComponentCheckResult;
898 }
899 return CheckResult;
900 }
901 if (Ty->isStructTy()) {
902 Value *CheckResult = nullptr;
903 for (auto I : seq(Size: Ty->getStructNumElements())) {
904 if (Config.getExtendedFPType(FT: Ty->getStructElementType(N: I)) == nullptr)
905 continue; // Only check FT values.
906 Value *ExtractV = Builder.CreateExtractValue(Agg: V, Idxs: I);
907 Value *ExtractShadowV = Builder.CreateExtractElement(Vec: ShadowV, Idx: I);
908 Value *ComponentCheckResult =
909 emitCheckInternal(V: ExtractV, ShadowV: ExtractShadowV, Builder, Loc);
910 CheckResult = CheckResult
911 ? Builder.CreateOr(LHS: CheckResult, RHS: ComponentCheckResult)
912 : ComponentCheckResult;
913 }
914 if (!CheckResult)
915 return ConstantInt::get(
916 Ty: Builder.getInt32Ty(),
917 V: static_cast<int>(ContinuationType::ContinueWithShadow));
918 return CheckResult;
919 }
920
921 llvm_unreachable("not implemented");
922}
923
924// Inserts a runtime check of V against its shadow value ShadowV.
925// We check values whenever they escape: on return, call, stores, and
926// insertvalue.
927// Returns the shadow value that should be used to continue the computations,
928// depending on the answer from the runtime.
929// TODO: Should we check on select ? phi ?
930Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
931 IRBuilder<> &Builder,
932 CheckLoc Loc) {
933 // Do not emit checks for constant values, this is redundant.
934 if (isa<Constant>(Val: V))
935 return ShadowV;
936
937 if (Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
938 Function *F = Inst->getFunction();
939 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName())) {
940 return ShadowV;
941 }
942 }
943
944 Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
945 Value *ICmpEQ = Builder.CreateICmpEQ(
946 LHS: CheckResult,
947 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
948 V: static_cast<int>(ContinuationType::ResumeFromValue)));
949 return Builder.CreateSelect(
950 C: ICmpEQ, True: Builder.CreateFPExt(V, DestTy: Config.getExtendedFPType(FT: V->getType())),
951 False: ShadowV);
952}
953
954// Inserts a check that fcmp on shadow values are consistent with that on base
955// values.
956void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
957 const ValueToShadowMap &Map) {
958 if (!Opts.nsan_instrument_fcmp)
959 return;
960
961 Function *F = FCmp.getFunction();
962 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(String: F->getName()))
963 return;
964
965 Value *LHS = FCmp.getOperand(i_nocapture: 0);
966 if (Config.getExtendedFPType(FT: LHS->getType()) == nullptr)
967 return;
968 Value *RHS = FCmp.getOperand(i_nocapture: 1);
969
970 // Split the basic block. On mismatch, we'll jump to the new basic block with
971 // a call to the runtime for error reporting.
972 BasicBlock *FCmpBB = FCmp.getParent();
973 BasicBlock *NextBB = FCmpBB->splitBasicBlock(I: FCmp.getNextNode());
974 // Remove the newly created terminator unconditional branch.
975 FCmpBB->back().eraseFromParent();
976 BasicBlock *FailBB =
977 BasicBlock::Create(Context, Name: "", Parent: FCmpBB->getParent(), InsertBefore: NextBB);
978
979 // Create the shadow fcmp and comparison between the fcmps.
980 IRBuilder<> FCmpBuilder(FCmpBB);
981 FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
982 Value *ShadowLHS = Map.getShadow(V: LHS);
983 Value *ShadowRHS = Map.getShadow(V: RHS);
984 // See the help text of -nsan-truncate-fcmp-eq.
985 if (FCmp.isEquality() && Opts.nsan_truncate_fcmp_eq) {
986 Type *Ty = ShadowLHS->getType();
987 ShadowLHS = FCmpBuilder.CreateFPExt(
988 V: FCmpBuilder.CreateFPTrunc(V: ShadowLHS, DestTy: LHS->getType()), DestTy: Ty);
989 ShadowRHS = FCmpBuilder.CreateFPExt(
990 V: FCmpBuilder.CreateFPTrunc(V: ShadowRHS, DestTy: RHS->getType()), DestTy: Ty);
991 }
992 Value *ShadowFCmp =
993 FCmpBuilder.CreateFCmp(P: FCmp.getPredicate(), LHS: ShadowLHS, RHS: ShadowRHS);
994 Value *OriginalAndShadowFcmpMatch =
995 FCmpBuilder.CreateICmpEQ(LHS: &FCmp, RHS: ShadowFCmp);
996
997 if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
998 // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
999 // where an element is true if the corresponding elements in original and
1000 // shadow are the same. We want all elements to be 1.
1001 OriginalAndShadowFcmpMatch =
1002 FCmpBuilder.CreateAndReduce(Src: OriginalAndShadowFcmpMatch);
1003 }
1004
1005 // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common
1006 // case.
1007 FCmpBuilder.CreateCondBr(Cond: OriginalAndShadowFcmpMatch, True: NextBB, False: FailBB,
1008 BranchWeights: MDBuilder(Context).createLikelyBranchWeights());
1009
1010 // Fill in FailBB.
1011 IRBuilder<> FailBuilder(FailBB);
1012 FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
1013
1014 const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
1015 &FailBuilder](Value *L, Value *R, Value *ShadowL,
1016 Value *ShadowR, Value *Result,
1017 Value *ShadowResult) {
1018 Type *FT = L->getType();
1019 FunctionCallee *Callee = nullptr;
1020 if (FT->isFloatTy()) {
1021 Callee = &(NsanFCmpFail[kFloat]);
1022 } else if (FT->isDoubleTy()) {
1023 Callee = &(NsanFCmpFail[kDouble]);
1024 } else if (FT->isX86_FP80Ty()) {
1025 // TODO: make NsanFCmpFailLongDouble work.
1026 Callee = &(NsanFCmpFail[kDouble]);
1027 L = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1028 R = FailBuilder.CreateFPTrunc(V: L, DestTy: Type::getDoubleTy(C&: Context));
1029 } else {
1030 llvm_unreachable("not implemented");
1031 }
1032 FailBuilder.CreateCall(Callee: *Callee, Args: {L, R, ShadowL, ShadowR,
1033 ConstantInt::get(Ty: FCmpBuilder.getInt32Ty(),
1034 V: FCmp.getPredicate()),
1035 Result, ShadowResult});
1036 };
1037 if (LHS->getType()->isVectorTy()) {
1038 for (int I = 0, E = cast<VectorType>(Val: LHS->getType())
1039 ->getElementCount()
1040 .getFixedValue();
1041 I < E; ++I) {
1042 Value *ExtractLHS = FailBuilder.CreateExtractElement(Vec: LHS, Idx: I);
1043 Value *ExtractRHS = FailBuilder.CreateExtractElement(Vec: RHS, Idx: I);
1044 Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(Vec: ShadowLHS, Idx: I);
1045 Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(Vec: ShadowRHS, Idx: I);
1046 Value *ExtractFCmp = FailBuilder.CreateExtractElement(Vec: &FCmp, Idx: I);
1047 Value *ExtractShadowFCmp =
1048 FailBuilder.CreateExtractElement(Vec: ShadowFCmp, Idx: I);
1049 EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,
1050 ExtractFCmp, ExtractShadowFCmp);
1051 }
1052 } else {
1053 EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
1054 }
1055 FailBuilder.CreateBr(Dest: NextBB);
1056
1057 ++NumInstrumentedFCmp;
1058}
1059
1060// Creates a shadow phi value for any phi that defines a value of FT type.
1061PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
1062 PHINode &Phi, const TargetLibraryInfo &TLI) {
1063 Type *VT = Phi.getType();
1064 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1065 if (ExtendedVT == nullptr)
1066 return nullptr; // Not an FT value.
1067 // The phi operands are shadow values and are not available when the phi is
1068 // created. They will be populated in a final phase, once all shadow values
1069 // have been created.
1070 PHINode *Shadow = PHINode::Create(Ty: ExtendedVT, NumReservedValues: Phi.getNumIncomingValues());
1071 Shadow->insertAfter(InsertPos: Phi.getIterator());
1072 return Shadow;
1073}
1074
1075Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
1076 Type *ExtendedVT) {
1077 IRBuilder<> Builder(Load.getNextNode());
1078 Builder.SetCurrentDebugLocation(Load.getDebugLoc());
1079 if (addrPointsToConstantData(Addr: Load.getPointerOperand())) {
1080 // No need to look into the shadow memory, the value is a constant. Just
1081 // convert from FT to 2FT.
1082 return Builder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1083 }
1084
1085 // if (%shadowptr == &)
1086 // %shadow = fpext %v
1087 // else
1088 // %shadow = load (ptrcast %shadow_ptr))
1089 // Considered options here:
1090 // - Have `NsanGetShadowPtrForLoad` return a fixed address
1091 // &__nsan_unknown_value_shadow_address that is valid to load from, and
1092 // use a select. This has the advantage that the generated IR is simpler.
1093 // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does
1094 // not short-circuit, dereferencing the returned pointer is no longer an
1095 // option, have to split and create a separate basic block. This has the
1096 // advantage of being easier to debug because it crashes if we ever mess
1097 // up.
1098
1099 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1100 Value *ShadowPtr = Builder.CreateCall(
1101 Callee: NsanGetShadowPtrForLoad[Extents.ValueType],
1102 Args: {Load.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1103 ++NumInstrumentedFTLoads;
1104
1105 // Split the basic block.
1106 BasicBlock *LoadBB = Load.getParent();
1107 BasicBlock *NextBB = LoadBB->splitBasicBlock(I: Builder.GetInsertPoint());
1108 // Create the two options for creating the shadow value.
1109 BasicBlock *ShadowLoadBB =
1110 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1111 BasicBlock *FExtBB =
1112 BasicBlock::Create(Context, Name: "", Parent: LoadBB->getParent(), InsertBefore: NextBB);
1113
1114 // Replace the newly created terminator unconditional branch by a conditional
1115 // branch to one of the options.
1116 {
1117 LoadBB->back().eraseFromParent();
1118 IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
1119 LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1120 LoadBBBuilder.CreateCondBr(Cond: LoadBBBuilder.CreateIsNull(Arg: ShadowPtr), True: FExtBB,
1121 False: ShadowLoadBB);
1122 }
1123
1124 // Fill in ShadowLoadBB.
1125 IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
1126 ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1127 Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
1128 Ty: ExtendedVT, Ptr: ShadowPtr, Align: Align(1), isVolatile: Load.isVolatile());
1129 if (Opts.nsan_check_loads) {
1130 ShadowLoad = emitCheck(V: &Load, ShadowV: ShadowLoad, Builder&: ShadowLoadBBBuilder,
1131 Loc: CheckLoc::makeLoad(Address: Load.getPointerOperand()));
1132 }
1133 ShadowLoadBBBuilder.CreateBr(Dest: NextBB);
1134
1135 // Fill in FExtBB.
1136 IRBuilder<> FExtBBBuilder(FExtBB);
1137 FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1138 Value *FExt = FExtBBBuilder.CreateFPExt(V: &Load, DestTy: ExtendedVT);
1139 FExtBBBuilder.CreateBr(Dest: NextBB);
1140
1141 // The shadow value come from any of the options.
1142 IRBuilder<> NextBBBuilder(&*NextBB->begin());
1143 NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
1144 PHINode *ShadowPhi = NextBBBuilder.CreatePHI(Ty: ExtendedVT, NumReservedValues: 2);
1145 ShadowPhi->addIncoming(V: ShadowLoad, BB: ShadowLoadBB);
1146 ShadowPhi->addIncoming(V: FExt, BB: FExtBB);
1147 return ShadowPhi;
1148}
1149
1150Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,
1151 Type *VT, Type *ExtendedVT,
1152 const ValueToShadowMap &Map,
1153 IRBuilder<> &Builder) {
1154 Value *OrigSource = Trunc.getOperand(i_nocapture: 0);
1155 Type *OrigSourceTy = OrigSource->getType();
1156 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1157
1158 // When truncating:
1159 // - (A) If the source has a shadow, we truncate from the shadow, else we
1160 // truncate from the original source.
1161 // - (B) If the shadow of the source is larger than the shadow of the dest,
1162 // we still need a truncate. Else, the shadow of the source is the same
1163 // type as the shadow of the dest (because mappings are non-decreasing), so
1164 // we don't need to emit a truncate.
1165 // Examples,
1166 // with a mapping of {f32->f64;f64->f80;f80->f128}
1167 // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double
1168 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1169 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1170 // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1)
1171 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80
1172 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1173 // with a mapping of {f32->f64;f64->f128;f80->f128}
1174 // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double
1175 // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
1176 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
1177 // fptrunc x86_fp80 %1 to double -> fp128 %1
1178 // fptrunc fp128 %1 to double -> fp128 %1
1179 // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
1180 // with a mapping of {f32->f32;f64->f32;f80->f64}
1181 // fptrunc double %1 to float -> float s(%1)
1182 // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float
1183 // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float
1184 // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float
1185 // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float
1186 // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double
1187
1188 // See (A) above.
1189 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1190 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1191 // See (B) above.
1192 if (SourceTy == ExtendedVT)
1193 return Source;
1194
1195 return Builder.CreateFPTrunc(V: Source, DestTy: ExtendedVT);
1196}
1197
1198Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,
1199 Type *ExtendedVT,
1200 const ValueToShadowMap &Map,
1201 IRBuilder<> &Builder) {
1202 Value *OrigSource = Ext.getOperand(i_nocapture: 0);
1203 Type *OrigSourceTy = OrigSource->getType();
1204 Type *ExtendedSourceTy = Config.getExtendedFPType(FT: OrigSourceTy);
1205 // When extending:
1206 // - (A) If the source has a shadow, we extend from the shadow, else we
1207 // extend from the original source.
1208 // - (B) If the shadow of the dest is larger than the shadow of the source,
1209 // we still need an extend. Else, the shadow of the source is the same
1210 // type as the shadow of the dest (because mappings are non-decreasing), so
1211 // we don't need to emit an extend.
1212 // Examples,
1213 // with a mapping of {f32->f64;f64->f80;f80->f128}
1214 // fpext half %1 to float -> fpext half %1 to double
1215 // fpext half %1 to double -> fpext half %1 to x86_fp80
1216 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1217 // fpext float %1 to double -> double s(%1)
1218 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1219 // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128
1220 // with a mapping of {f32->f64;f64->f128;f80->f128}
1221 // fpext half %1 to float -> fpext half %1 to double
1222 // fpext half %1 to double -> fpext half %1 to fp128
1223 // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
1224 // fpext float %1 to double -> fpext double s(%1) to fp128
1225 // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
1226 // fpext double %1 to x86_fp80 -> fp128 s(%1)
1227 // with a mapping of {f32->f32;f64->f32;f80->f64}
1228 // fpext half %1 to float -> fpext half %1 to float
1229 // fpext half %1 to double -> fpext half %1 to float
1230 // fpext half %1 to x86_fp80 -> fpext half %1 to double
1231 // fpext float %1 to double -> s(%1)
1232 // fpext float %1 to x86_fp80 -> fpext float s(%1) to double
1233 // fpext double %1 to x86_fp80 -> fpext float s(%1) to double
1234
1235 // See (A) above.
1236 Value *Source = ExtendedSourceTy ? Map.getShadow(V: OrigSource) : OrigSource;
1237 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
1238 // See (B) above.
1239 if (SourceTy == ExtendedVT)
1240 return Source;
1241
1242 return Builder.CreateFPExt(V: Source, DestTy: ExtendedVT);
1243}
1244
1245namespace {
1246// TODO: This should be tablegen-ed.
1247struct KnownIntrinsic {
1248 struct WidenedIntrinsic {
1249 const char *NarrowName;
1250 Intrinsic::ID ID; // wide id.
1251 using FnTypeFactory = FunctionType *(*)(LLVMContext &);
1252 FnTypeFactory MakeFnTy;
1253 };
1254
1255 static const char *get(LibFunc LFunc);
1256
1257 // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
1258 // that applies the same operation on the shadow argument.
1259 // Options are:
1260 // - pass in the ID and full function type,
1261 // - pass in the name, which includes the function type through mangling.
1262 static const WidenedIntrinsic *widen(StringRef Name);
1263
1264private:
1265 struct LFEntry {
1266 LibFunc LFunc;
1267 const char *IntrinsicName;
1268 };
1269 static const LFEntry kLibfuncIntrinsics[];
1270
1271 static const WidenedIntrinsic kWidenedIntrinsics[];
1272};
1273} // namespace
1274
1275static FunctionType *makeDoubleDouble(LLVMContext &C) {
1276 return FunctionType::get(Result: Type::getDoubleTy(C), Params: {Type::getDoubleTy(C)}, isVarArg: false);
1277}
1278
1279static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {
1280 return FunctionType::get(Result: Type::getX86_FP80Ty(C), Params: {Type::getX86_FP80Ty(C)},
1281 isVarArg: false);
1282}
1283
1284static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {
1285 return FunctionType::get(Result: Type::getDoubleTy(C),
1286 Params: {Type::getDoubleTy(C), Type::getInt32Ty(C)}, isVarArg: false);
1287}
1288
1289static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {
1290 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1291 Params: {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
1292 isVarArg: false);
1293}
1294
1295static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {
1296 return FunctionType::get(Result: Type::getDoubleTy(C),
1297 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C)}, isVarArg: false);
1298}
1299
1300static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {
1301 return FunctionType::get(Result: Type::getX86_FP80Ty(C),
1302 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1303 isVarArg: false);
1304}
1305
1306static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {
1307 return FunctionType::get(
1308 Result: Type::getDoubleTy(C),
1309 Params: {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
1310 isVarArg: false);
1311}
1312
1313static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {
1314 return FunctionType::get(
1315 Result: Type::getX86_FP80Ty(C),
1316 Params: {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
1317 isVarArg: false);
1318}
1319
1320const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
1321 // TODO: Right now we ignore vector intrinsics.
1322 // This is hard because we have to model the semantics of the intrinsics,
1323 // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
1324 // Intrinsics that take any non-vector FT types:
1325 // NOTE: Right now because of
1326 // https://github.com/llvm/llvm-project/issues/44744
1327 // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and
1328 // come back).
1329 {.NarrowName: "llvm.sqrt.f32", .ID: Intrinsic::sqrt, .MakeFnTy: makeDoubleDouble},
1330 {.NarrowName: "llvm.sqrt.f64", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1331 {.NarrowName: "llvm.sqrt.f80", .ID: Intrinsic::sqrt, .MakeFnTy: makeX86FP80X86FP80},
1332 {.NarrowName: "llvm.powi.f32", .ID: Intrinsic::powi, .MakeFnTy: makeDoubleDoubleI32},
1333 {.NarrowName: "llvm.powi.f64", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1334 {.NarrowName: "llvm.powi.f80", .ID: Intrinsic::powi, .MakeFnTy: makeX86FP80X86FP80I32},
1335 {.NarrowName: "llvm.sin.f32", .ID: Intrinsic::sin, .MakeFnTy: makeDoubleDouble},
1336 {.NarrowName: "llvm.sin.f64", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1337 {.NarrowName: "llvm.sin.f80", .ID: Intrinsic::sin, .MakeFnTy: makeX86FP80X86FP80},
1338 {.NarrowName: "llvm.cos.f32", .ID: Intrinsic::cos, .MakeFnTy: makeDoubleDouble},
1339 {.NarrowName: "llvm.cos.f64", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1340 {.NarrowName: "llvm.cos.f80", .ID: Intrinsic::cos, .MakeFnTy: makeX86FP80X86FP80},
1341 {.NarrowName: "llvm.pow.f32", .ID: Intrinsic::pow, .MakeFnTy: makeDoubleDoubleDouble},
1342 {.NarrowName: "llvm.pow.f64", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1343 {.NarrowName: "llvm.pow.f80", .ID: Intrinsic::pow, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1344 {.NarrowName: "llvm.exp.f32", .ID: Intrinsic::exp, .MakeFnTy: makeDoubleDouble},
1345 {.NarrowName: "llvm.exp.f64", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1346 {.NarrowName: "llvm.exp.f80", .ID: Intrinsic::exp, .MakeFnTy: makeX86FP80X86FP80},
1347 {.NarrowName: "llvm.exp2.f32", .ID: Intrinsic::exp2, .MakeFnTy: makeDoubleDouble},
1348 {.NarrowName: "llvm.exp2.f64", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1349 {.NarrowName: "llvm.exp2.f80", .ID: Intrinsic::exp2, .MakeFnTy: makeX86FP80X86FP80},
1350 {.NarrowName: "llvm.log.f32", .ID: Intrinsic::log, .MakeFnTy: makeDoubleDouble},
1351 {.NarrowName: "llvm.log.f64", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1352 {.NarrowName: "llvm.log.f80", .ID: Intrinsic::log, .MakeFnTy: makeX86FP80X86FP80},
1353 {.NarrowName: "llvm.log10.f32", .ID: Intrinsic::log10, .MakeFnTy: makeDoubleDouble},
1354 {.NarrowName: "llvm.log10.f64", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1355 {.NarrowName: "llvm.log10.f80", .ID: Intrinsic::log10, .MakeFnTy: makeX86FP80X86FP80},
1356 {.NarrowName: "llvm.log2.f32", .ID: Intrinsic::log2, .MakeFnTy: makeDoubleDouble},
1357 {.NarrowName: "llvm.log2.f64", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1358 {.NarrowName: "llvm.log2.f80", .ID: Intrinsic::log2, .MakeFnTy: makeX86FP80X86FP80},
1359 {.NarrowName: "llvm.fma.f32", .ID: Intrinsic::fma, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1360 {.NarrowName: "llvm.fma.f64", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1361 {.NarrowName: "llvm.fma.f80", .ID: Intrinsic::fma, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1362 {.NarrowName: "llvm.fmuladd.f32", .ID: Intrinsic::fmuladd, .MakeFnTy: makeDoubleDoubleDoubleDouble},
1363 {.NarrowName: "llvm.fmuladd.f64", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1364 {.NarrowName: "llvm.fmuladd.f80", .ID: Intrinsic::fmuladd, .MakeFnTy: makeX86FP80X86FP80X86FP80X86FP80},
1365 {.NarrowName: "llvm.fabs.f32", .ID: Intrinsic::fabs, .MakeFnTy: makeDoubleDouble},
1366 {.NarrowName: "llvm.fabs.f64", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1367 {.NarrowName: "llvm.fabs.f80", .ID: Intrinsic::fabs, .MakeFnTy: makeX86FP80X86FP80},
1368 {.NarrowName: "llvm.minnum.f32", .ID: Intrinsic::minnum, .MakeFnTy: makeDoubleDoubleDouble},
1369 {.NarrowName: "llvm.minnum.f64", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1370 {.NarrowName: "llvm.minnum.f80", .ID: Intrinsic::minnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1371 {.NarrowName: "llvm.maxnum.f32", .ID: Intrinsic::maxnum, .MakeFnTy: makeDoubleDoubleDouble},
1372 {.NarrowName: "llvm.maxnum.f64", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1373 {.NarrowName: "llvm.maxnum.f80", .ID: Intrinsic::maxnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1374 {.NarrowName: "llvm.minimum.f32", .ID: Intrinsic::minimum, .MakeFnTy: makeDoubleDoubleDouble},
1375 {.NarrowName: "llvm.minimum.f64", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1376 {.NarrowName: "llvm.minimum.f80", .ID: Intrinsic::minimum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1377 {.NarrowName: "llvm.maximum.f32", .ID: Intrinsic::maximum, .MakeFnTy: makeDoubleDoubleDouble},
1378 {.NarrowName: "llvm.maximum.f64", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1379 {.NarrowName: "llvm.maximum.f80", .ID: Intrinsic::maximum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1380 {.NarrowName: "llvm.minimumnum.f32", .ID: Intrinsic::minimumnum, .MakeFnTy: makeDoubleDoubleDouble},
1381 {.NarrowName: "llvm.minimumnum.f64", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1382 {.NarrowName: "llvm.minimumnum.f80", .ID: Intrinsic::minimumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1383 {.NarrowName: "llvm.maximumnum.f32", .ID: Intrinsic::maximumnum, .MakeFnTy: makeDoubleDoubleDouble},
1384 {.NarrowName: "llvm.maximumnum.f64", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1385 {.NarrowName: "llvm.maximumnum.f80", .ID: Intrinsic::maximumnum, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1386 {.NarrowName: "llvm.copysign.f32", .ID: Intrinsic::copysign, .MakeFnTy: makeDoubleDoubleDouble},
1387 {.NarrowName: "llvm.copysign.f64", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1388 {.NarrowName: "llvm.copysign.f80", .ID: Intrinsic::copysign, .MakeFnTy: makeX86FP80X86FP80X86FP80},
1389 {.NarrowName: "llvm.floor.f32", .ID: Intrinsic::floor, .MakeFnTy: makeDoubleDouble},
1390 {.NarrowName: "llvm.floor.f64", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1391 {.NarrowName: "llvm.floor.f80", .ID: Intrinsic::floor, .MakeFnTy: makeX86FP80X86FP80},
1392 {.NarrowName: "llvm.ceil.f32", .ID: Intrinsic::ceil, .MakeFnTy: makeDoubleDouble},
1393 {.NarrowName: "llvm.ceil.f64", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1394 {.NarrowName: "llvm.ceil.f80", .ID: Intrinsic::ceil, .MakeFnTy: makeX86FP80X86FP80},
1395 {.NarrowName: "llvm.trunc.f32", .ID: Intrinsic::trunc, .MakeFnTy: makeDoubleDouble},
1396 {.NarrowName: "llvm.trunc.f64", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1397 {.NarrowName: "llvm.trunc.f80", .ID: Intrinsic::trunc, .MakeFnTy: makeX86FP80X86FP80},
1398 {.NarrowName: "llvm.rint.f32", .ID: Intrinsic::rint, .MakeFnTy: makeDoubleDouble},
1399 {.NarrowName: "llvm.rint.f64", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1400 {.NarrowName: "llvm.rint.f80", .ID: Intrinsic::rint, .MakeFnTy: makeX86FP80X86FP80},
1401 {.NarrowName: "llvm.nearbyint.f32", .ID: Intrinsic::nearbyint, .MakeFnTy: makeDoubleDouble},
1402 {.NarrowName: "llvm.nearbyint.f64", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1403 {.NarrowName: "llvm.nearbyint.f80", .ID: Intrinsic::nearbyint, .MakeFnTy: makeX86FP80X86FP80},
1404 {.NarrowName: "llvm.round.f32", .ID: Intrinsic::round, .MakeFnTy: makeDoubleDouble},
1405 {.NarrowName: "llvm.round.f64", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1406 {.NarrowName: "llvm.round.f80", .ID: Intrinsic::round, .MakeFnTy: makeX86FP80X86FP80},
1407};
1408
1409const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {
1410 {.LFunc: LibFunc_sqrtf, .IntrinsicName: "llvm.sqrt.f32"},
1411 {.LFunc: LibFunc_sqrt, .IntrinsicName: "llvm.sqrt.f64"},
1412 {.LFunc: LibFunc_sqrtl, .IntrinsicName: "llvm.sqrt.f80"},
1413 {.LFunc: LibFunc_sinf, .IntrinsicName: "llvm.sin.f32"},
1414 {.LFunc: LibFunc_sin, .IntrinsicName: "llvm.sin.f64"},
1415 {.LFunc: LibFunc_sinl, .IntrinsicName: "llvm.sin.f80"},
1416 {.LFunc: LibFunc_cosf, .IntrinsicName: "llvm.cos.f32"},
1417 {.LFunc: LibFunc_cos, .IntrinsicName: "llvm.cos.f64"},
1418 {.LFunc: LibFunc_cosl, .IntrinsicName: "llvm.cos.f80"},
1419 {.LFunc: LibFunc_powf, .IntrinsicName: "llvm.pow.f32"},
1420 {.LFunc: LibFunc_pow, .IntrinsicName: "llvm.pow.f64"},
1421 {.LFunc: LibFunc_powl, .IntrinsicName: "llvm.pow.f80"},
1422 {.LFunc: LibFunc_expf, .IntrinsicName: "llvm.exp.f32"},
1423 {.LFunc: LibFunc_exp, .IntrinsicName: "llvm.exp.f64"},
1424 {.LFunc: LibFunc_expl, .IntrinsicName: "llvm.exp.f80"},
1425 {.LFunc: LibFunc_exp2f, .IntrinsicName: "llvm.exp2.f32"},
1426 {.LFunc: LibFunc_exp2, .IntrinsicName: "llvm.exp2.f64"},
1427 {.LFunc: LibFunc_exp2l, .IntrinsicName: "llvm.exp2.f80"},
1428 {.LFunc: LibFunc_logf, .IntrinsicName: "llvm.log.f32"},
1429 {.LFunc: LibFunc_log, .IntrinsicName: "llvm.log.f64"},
1430 {.LFunc: LibFunc_logl, .IntrinsicName: "llvm.log.f80"},
1431 {.LFunc: LibFunc_log10f, .IntrinsicName: "llvm.log10.f32"},
1432 {.LFunc: LibFunc_log10, .IntrinsicName: "llvm.log10.f64"},
1433 {.LFunc: LibFunc_log10l, .IntrinsicName: "llvm.log10.f80"},
1434 {.LFunc: LibFunc_log2f, .IntrinsicName: "llvm.log2.f32"},
1435 {.LFunc: LibFunc_log2, .IntrinsicName: "llvm.log2.f64"},
1436 {.LFunc: LibFunc_log2l, .IntrinsicName: "llvm.log2.f80"},
1437 {.LFunc: LibFunc_fabsf, .IntrinsicName: "llvm.fabs.f32"},
1438 {.LFunc: LibFunc_fabs, .IntrinsicName: "llvm.fabs.f64"},
1439 {.LFunc: LibFunc_fabsl, .IntrinsicName: "llvm.fabs.f80"},
1440 {.LFunc: LibFunc_copysignf, .IntrinsicName: "llvm.copysign.f32"},
1441 {.LFunc: LibFunc_copysign, .IntrinsicName: "llvm.copysign.f64"},
1442 {.LFunc: LibFunc_copysignl, .IntrinsicName: "llvm.copysign.f80"},
1443 {.LFunc: LibFunc_floorf, .IntrinsicName: "llvm.floor.f32"},
1444 {.LFunc: LibFunc_floor, .IntrinsicName: "llvm.floor.f64"},
1445 {.LFunc: LibFunc_floorl, .IntrinsicName: "llvm.floor.f80"},
1446 {.LFunc: LibFunc_fmaxf, .IntrinsicName: "llvm.maxnum.f32"},
1447 {.LFunc: LibFunc_fmax, .IntrinsicName: "llvm.maxnum.f64"},
1448 {.LFunc: LibFunc_fmaxl, .IntrinsicName: "llvm.maxnum.f80"},
1449 {.LFunc: LibFunc_fminf, .IntrinsicName: "llvm.minnum.f32"},
1450 {.LFunc: LibFunc_fmin, .IntrinsicName: "llvm.minnum.f64"},
1451 {.LFunc: LibFunc_fminl, .IntrinsicName: "llvm.minnum.f80"},
1452 {.LFunc: LibFunc_fmaximum_numf, .IntrinsicName: "llvm.maximumnum.f32"},
1453 {.LFunc: LibFunc_fmaximum_num, .IntrinsicName: "llvm.maximumnum.f64"},
1454 {.LFunc: LibFunc_fmaximum_numl, .IntrinsicName: "llvm.maximumnum.f80"},
1455 {.LFunc: LibFunc_fminimum_numf, .IntrinsicName: "llvm.minimumnum.f32"},
1456 {.LFunc: LibFunc_fminimum_num, .IntrinsicName: "llvm.minimumnum.f64"},
1457 {.LFunc: LibFunc_fminimum_numl, .IntrinsicName: "llvm.minimumnum.f80"},
1458 {.LFunc: LibFunc_ceilf, .IntrinsicName: "llvm.ceil.f32"},
1459 {.LFunc: LibFunc_ceil, .IntrinsicName: "llvm.ceil.f64"},
1460 {.LFunc: LibFunc_ceill, .IntrinsicName: "llvm.ceil.f80"},
1461 {.LFunc: LibFunc_truncf, .IntrinsicName: "llvm.trunc.f32"},
1462 {.LFunc: LibFunc_trunc, .IntrinsicName: "llvm.trunc.f64"},
1463 {.LFunc: LibFunc_truncl, .IntrinsicName: "llvm.trunc.f80"},
1464 {.LFunc: LibFunc_rintf, .IntrinsicName: "llvm.rint.f32"},
1465 {.LFunc: LibFunc_rint, .IntrinsicName: "llvm.rint.f64"},
1466 {.LFunc: LibFunc_rintl, .IntrinsicName: "llvm.rint.f80"},
1467 {.LFunc: LibFunc_nearbyintf, .IntrinsicName: "llvm.nearbyint.f32"},
1468 {.LFunc: LibFunc_nearbyint, .IntrinsicName: "llvm.nearbyint.f64"},
1469 {.LFunc: LibFunc_nearbyintl, .IntrinsicName: "llvm.nearbyint.f80"},
1470 {.LFunc: LibFunc_roundf, .IntrinsicName: "llvm.round.f32"},
1471 {.LFunc: LibFunc_round, .IntrinsicName: "llvm.round.f64"},
1472 {.LFunc: LibFunc_roundl, .IntrinsicName: "llvm.round.f80"},
1473};
1474
1475const char *KnownIntrinsic::get(LibFunc LFunc) {
1476 for (const auto &E : kLibfuncIntrinsics) {
1477 if (E.LFunc == LFunc)
1478 return E.IntrinsicName;
1479 }
1480 return nullptr;
1481}
1482
1483const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {
1484 for (const auto &E : kWidenedIntrinsics) {
1485 if (E.NarrowName == Name)
1486 return &E;
1487 }
1488 return nullptr;
1489}
1490
1491// Returns the name of the LLVM intrinsic corresponding to the given function.
1492static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,
1493 const TargetLibraryInfo &TLI) {
1494 LibFunc LFunc = TLI.getLibFunc(FDecl: Fn);
1495 if (LFunc == NotLibFunc)
1496 return nullptr;
1497
1498 if (const char *Name = KnownIntrinsic::get(LFunc))
1499 return Name;
1500
1501 LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");
1502 return nullptr;
1503}
1504
1505// Try to handle a known function call.
1506Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(
1507 CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,
1508 const ValueToShadowMap &Map, IRBuilder<> &Builder) {
1509 Function *Fn = Call.getCalledFunction();
1510 if (Fn == nullptr)
1511 return nullptr;
1512
1513 Intrinsic::ID WidenedId = Intrinsic::ID();
1514 FunctionType *WidenedFnTy = nullptr;
1515 if (const auto ID = Fn->getIntrinsicID()) {
1516 const auto *Widened = KnownIntrinsic::widen(Name: Fn->getName());
1517 if (Widened) {
1518 WidenedId = Widened->ID;
1519 WidenedFnTy = Widened->MakeFnTy(Context);
1520 } else {
1521 // If we don't know how to widen the intrinsic, we have no choice but to
1522 // call the non-wide version on a truncated shadow and extend again
1523 // afterwards.
1524 WidenedId = ID;
1525 WidenedFnTy = Fn->getFunctionType();
1526 }
1527 } else if (const char *Name = getIntrinsicFromLibfunc(Fn&: *Fn, VT, TLI)) {
1528 // We might have a call to a library function that we can replace with a
1529 // wider Intrinsic.
1530 const auto *Widened = KnownIntrinsic::widen(Name);
1531 assert(Widened && "make sure KnownIntrinsic entries are consistent");
1532 WidenedId = Widened->ID;
1533 WidenedFnTy = Widened->MakeFnTy(Context);
1534 } else {
1535 // This is not a known library function or intrinsic.
1536 return nullptr;
1537 }
1538
1539 // Check that the widened intrinsic is valid.
1540 SmallVector<Type *, 4> OverloadTys;
1541 [[maybe_unused]] bool IsValid =
1542 Intrinsic::isSignatureValid(ID: WidenedId, FT: WidenedFnTy, OverloadTys);
1543 assert(IsValid && "invalid widened intrinsic");
1544 // For known intrinsic functions, we create a second call to the same
1545 // intrinsic with a different type.
1546 SmallVector<Value *, 4> Args;
1547 // The last operand is the intrinsic itself, skip it.
1548 for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {
1549 Value *Arg = Call.getOperand(i_nocapture: I);
1550 Type *OrigArgTy = Arg->getType();
1551 Type *IntrinsicArgTy = WidenedFnTy->getParamType(i: I);
1552 if (OrigArgTy == IntrinsicArgTy) {
1553 Args.push_back(Elt: Arg); // The arg is passed as is.
1554 continue;
1555 }
1556 Type *ShadowArgTy = Config.getExtendedFPType(FT: Arg->getType());
1557 assert(ShadowArgTy &&
1558 "don't know how to get the shadow value for a non-FT");
1559 Value *Shadow = Map.getShadow(V: Arg);
1560 if (ShadowArgTy == IntrinsicArgTy) {
1561 // The shadow is the right type for the intrinsic.
1562 assert(Shadow->getType() == ShadowArgTy);
1563 Args.push_back(Elt: Shadow);
1564 continue;
1565 }
1566 // There is no intrinsic with his level of precision, truncate the shadow.
1567 Args.push_back(Elt: Builder.CreateFPTrunc(V: Shadow, DestTy: IntrinsicArgTy));
1568 }
1569 Value *IntrinsicCall = Builder.CreateIntrinsic(ID: WidenedId, OverloadTypes: OverloadTys, Args);
1570 return WidenedFnTy->getReturnType() == ExtendedVT
1571 ? IntrinsicCall
1572 : Builder.CreateFPExt(V: IntrinsicCall, DestTy: ExtendedVT);
1573}
1574
1575// Handle a CallBase, i.e. a function call, an inline asm sequence, or an
1576// invoke.
1577Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,
1578 Type *ExtendedVT,
1579 const TargetLibraryInfo &TLI,
1580 const ValueToShadowMap &Map,
1581 IRBuilder<> &Builder) {
1582 // We cannot look inside inline asm, just expand the result again.
1583 if (Call.isInlineAsm())
1584 return Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT);
1585
1586 // Intrinsics and library functions (e.g. sin, exp) are handled
1587 // specifically, because we know their semantics and can do better than
1588 // blindly calling them (e.g. compute the sinus in the actual shadow domain).
1589 if (Value *V =
1590 maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))
1591 return V;
1592
1593 // If the return tag matches that of the called function, read the extended
1594 // return value from the shadow ret ptr. Else, just extend the return value.
1595 Value *L =
1596 Builder.CreateLoad(Ty: IntptrTy, Ptr: NsanShadowRetTag, /*isVolatile=*/false);
1597 Value *HasShadowRet = Builder.CreateICmpEQ(
1598 LHS: L, RHS: Builder.CreatePtrToInt(V: Call.getCalledOperand(), DestTy: IntptrTy));
1599
1600 Value *ShadowRetVal = Builder.CreateLoad(
1601 Ty: ExtendedVT,
1602 Ptr: Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0),
1603 /*isVolatile=*/false);
1604 Value *Shadow = Builder.CreateSelect(C: HasShadowRet, True: ShadowRetVal,
1605 False: Builder.CreateFPExt(V: &Call, DestTy: ExtendedVT));
1606 ++NumInstrumentedFTCalls;
1607 return Shadow;
1608}
1609
1610// Creates a shadow value for the given FT value. At that point all operands are
1611// guaranteed to be available.
1612Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(
1613 Instruction &Inst, const TargetLibraryInfo &TLI,
1614 const ValueToShadowMap &Map) {
1615 Type *VT = Inst.getType();
1616 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1617 assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");
1618
1619 if (auto *Load = dyn_cast<LoadInst>(Val: &Inst))
1620 return handleLoad(Load&: *Load, VT, ExtendedVT);
1621
1622 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
1623 // Insert after the call.
1624 BasicBlock::iterator It(Inst);
1625 IRBuilder<> Builder(++It);
1626 Builder.SetCurrentDebugLocation(Call->getDebugLoc());
1627 return handleCallBase(Call&: *Call, VT, ExtendedVT, TLI, Map, Builder);
1628 }
1629
1630 if (auto *Invoke = dyn_cast<InvokeInst>(Val: &Inst)) {
1631 // The Invoke terminates the basic block, create a new basic block in
1632 // between the successful invoke and the next block.
1633 BasicBlock *InvokeBB = Invoke->getParent();
1634 BasicBlock *NextBB = Invoke->getNormalDest();
1635 BasicBlock *NewBB =
1636 BasicBlock::Create(Context, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
1637 Inst.replaceSuccessorWith(OldBB: NextBB, NewBB);
1638
1639 IRBuilder<> Builder(NewBB);
1640 Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());
1641 Value *Shadow = handleCallBase(Call&: *Invoke, VT, ExtendedVT, TLI, Map, Builder);
1642 Builder.CreateBr(Dest: NextBB);
1643 NewBB->replaceSuccessorsPhiUsesWith(Old: InvokeBB, New: NewBB);
1644 return Shadow;
1645 }
1646
1647 IRBuilder<> Builder(Inst.getNextNode());
1648 Builder.SetCurrentDebugLocation(Inst.getDebugLoc());
1649
1650 if (auto *Trunc = dyn_cast<FPTruncInst>(Val: &Inst))
1651 return handleTrunc(Trunc: *Trunc, VT, ExtendedVT, Map, Builder);
1652 if (auto *Ext = dyn_cast<FPExtInst>(Val: &Inst))
1653 return handleExt(Ext: *Ext, VT, ExtendedVT, Map, Builder);
1654
1655 if (auto *UnaryOp = dyn_cast<UnaryOperator>(Val: &Inst))
1656 return Builder.CreateUnOp(Opc: UnaryOp->getOpcode(),
1657 V: Map.getShadow(V: UnaryOp->getOperand(i_nocapture: 0)));
1658
1659 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: &Inst))
1660 return Builder.CreateBinOp(Opc: BinOp->getOpcode(),
1661 LHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 0)),
1662 RHS: Map.getShadow(V: BinOp->getOperand(i_nocapture: 1)));
1663
1664 if (isa<UIToFPInst>(Val: &Inst) || isa<SIToFPInst>(Val: &Inst)) {
1665 auto *Cast = cast<CastInst>(Val: &Inst);
1666 return Builder.CreateCast(Op: Cast->getOpcode(), V: Cast->getOperand(i_nocapture: 0),
1667 DestTy: ExtendedVT);
1668 }
1669
1670 if (auto *S = dyn_cast<SelectInst>(Val: &Inst))
1671 return Builder.CreateSelect(C: S->getCondition(),
1672 True: Map.getShadow(V: S->getTrueValue()),
1673 False: Map.getShadow(V: S->getFalseValue()));
1674
1675 if (auto *Freeze = dyn_cast<FreezeInst>(Val: &Inst))
1676 return Builder.CreateFreeze(V: Map.getShadow(V: Freeze->getOperand(i_nocapture: 0)));
1677
1678 if (auto *Extract = dyn_cast<ExtractElementInst>(Val: &Inst))
1679 return Builder.CreateExtractElement(
1680 Vec: Map.getShadow(V: Extract->getVectorOperand()), Idx: Extract->getIndexOperand());
1681
1682 if (auto *Insert = dyn_cast<InsertElementInst>(Val: &Inst))
1683 return Builder.CreateInsertElement(Vec: Map.getShadow(V: Insert->getOperand(i_nocapture: 0)),
1684 NewElt: Map.getShadow(V: Insert->getOperand(i_nocapture: 1)),
1685 Idx: Insert->getOperand(i_nocapture: 2));
1686
1687 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: &Inst))
1688 return Builder.CreateShuffleVector(V1: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 0)),
1689 V2: Map.getShadow(V: Shuffle->getOperand(i_nocapture: 1)),
1690 Mask: Shuffle->getShuffleMask());
1691 // TODO: We could make aggregate object first class citizens. For now we
1692 // just extend the extracted value.
1693 if (auto *Extract = dyn_cast<ExtractValueInst>(Val: &Inst))
1694 return Builder.CreateFPExt(V: Extract, DestTy: ExtendedVT);
1695
1696 if (auto *BC = dyn_cast<BitCastInst>(Val: &Inst))
1697 return Builder.CreateFPExt(V: BC, DestTy: ExtendedVT);
1698
1699 report_fatal_error(reason: "Unimplemented support for " +
1700 Twine(Inst.getOpcodeName()));
1701}
1702
1703// Creates a shadow value for an instruction that defines a value of FT type.
1704// FT operands that do not already have shadow values are created recursively.
1705// The DFS is guaranteed to not loop as phis and arguments already have
1706// shadows.
1707void NumericalStabilitySanitizer::maybeCreateShadowValue(
1708 Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
1709 Type *VT = Root.getType();
1710 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1711 if (ExtendedVT == nullptr)
1712 return; // Not an FT value.
1713
1714 if (Map.hasShadow(V: &Root))
1715 return; // Shadow already exists.
1716
1717 assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");
1718
1719 std::vector<Instruction *> DfsStack(1, &Root);
1720 while (!DfsStack.empty()) {
1721 // Ensure that all operands to the instruction have shadows before
1722 // proceeding.
1723 Instruction *I = DfsStack.back();
1724 // The shadow for the instruction might have been created deeper in the DFS,
1725 // see `forward_use_with_two_uses` test.
1726 if (Map.hasShadow(V: I)) {
1727 DfsStack.pop_back();
1728 continue;
1729 }
1730
1731 bool MissingShadow = false;
1732 for (Value *Op : I->operands()) {
1733 Type *VT = Op->getType();
1734 if (!Config.getExtendedFPType(FT: VT))
1735 continue; // Not an FT value.
1736 if (Map.hasShadow(V: Op))
1737 continue; // Shadow is already available.
1738 MissingShadow = true;
1739 DfsStack.push_back(x: cast<Instruction>(Val: Op));
1740 }
1741 if (MissingShadow)
1742 continue; // Process operands and come back to this instruction later.
1743
1744 // All operands have shadows. Create a shadow for the current value.
1745 Value *Shadow = createShadowValueWithOperandsAvailable(Inst&: *I, TLI, Map);
1746 Map.setShadow(V&: *I, Shadow&: *Shadow);
1747 DfsStack.pop_back();
1748 }
1749}
1750
1751// A floating-point store needs its value and type written to shadow memory.
1752void NumericalStabilitySanitizer::propagateFTStore(
1753 StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {
1754 Value *StoredValue = Store.getValueOperand();
1755 IRBuilder<> Builder(&Store);
1756 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1757 const auto Extents = getMemoryExtentsOrDie(FT: VT);
1758 Value *ShadowPtr = Builder.CreateCall(
1759 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1760 Args: {Store.getPointerOperand(), ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1761
1762 Value *StoredShadow = Map.getShadow(V: StoredValue);
1763 if (!Store.getParent()->getParent()->hasOptNone()) {
1764 // Only check stores when optimizing, because non-optimized code generates
1765 // too many stores to the stack, creating false positives.
1766 if (Opts.nsan_check_stores) {
1767 StoredShadow = emitCheck(V: StoredValue, ShadowV: StoredShadow, Builder,
1768 Loc: CheckLoc::makeStore(Address: Store.getPointerOperand()));
1769 ++NumInstrumentedFTStores;
1770 }
1771 }
1772
1773 Builder.CreateAlignedStore(Val: StoredShadow, Ptr: ShadowPtr, Align: Align(1),
1774 isVolatile: Store.isVolatile());
1775}
1776
1777// A non-ft store needs to invalidate shadow memory. Exceptions are:
1778// - memory transfers of floating-point data through other pointer types (llvm
1779// optimization passes transform `*(float*)a = *(float*)b` into
1780// `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.
1781// - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted
1782// ints. Note that this is not really necessary because if the value is
1783// unknown the framework will re-extend it on load anyway. It just felt
1784// easier to debug tests with vectors of FTs.
1785void NumericalStabilitySanitizer::propagateNonFTStore(
1786 StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {
1787 Value *PtrOp = Store.getPointerOperand();
1788 IRBuilder<> Builder(Store.getNextNode());
1789 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1790 Value *Dst = PtrOp;
1791 TypeSize SlotSize = DL.getTypeStoreSize(Ty: VT);
1792 assert(!SlotSize.isScalable() && "unsupported");
1793 const auto LoadSizeBytes = SlotSize.getFixedValue();
1794 Value *ValueSize = Constant::getIntegerValue(
1795 Ty: IntptrTy, V: APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));
1796
1797 ++NumInstrumentedNonFTStores;
1798 Value *StoredValue = Store.getValueOperand();
1799 if (LoadInst *Load = dyn_cast<LoadInst>(Val: StoredValue)) {
1800 // TODO: Handle the case when the value is from a phi.
1801 // This is a memory transfer with memcpy semantics. Copy the type and
1802 // value from the source. Note that we cannot use __nsan_copy_values()
1803 // here, because that will not work when there is a write to memory in
1804 // between the load and the store, e.g. in the case of a swap.
1805 Type *ShadowTypeIntTy = Type::getIntNTy(C&: Context, N: 8 * LoadSizeBytes);
1806 Type *ShadowValueIntTy =
1807 Type::getIntNTy(C&: Context, N: 8 * kShadowScale * LoadSizeBytes);
1808 IRBuilder<> LoadBuilder(Load->getNextNode());
1809 Builder.SetCurrentDebugLocation(Store.getDebugLoc());
1810 Value *LoadSrc = Load->getPointerOperand();
1811 // Read the shadow type and value at load time. The type has the same size
1812 // as the FT value, the value has twice its size.
1813 // TODO: cache them to avoid re-creating them when a load is used by
1814 // several stores. Maybe create them like the FT shadows when a load is
1815 // encountered.
1816 Value *RawShadowType = LoadBuilder.CreateAlignedLoad(
1817 Ty: ShadowTypeIntTy,
1818 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {LoadSrc}), Align: Align(1),
1819 /*isVolatile=*/false);
1820 Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(
1821 Ty: ShadowValueIntTy,
1822 Ptr: LoadBuilder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {LoadSrc}), Align: Align(1),
1823 /*isVolatile=*/false);
1824
1825 // Write back the shadow type and value at store time.
1826 Builder.CreateAlignedStore(
1827 Val: RawShadowType, Ptr: Builder.CreateCall(Callee: NsanGetRawShadowTypePtr, Args: {Dst}),
1828 Align: Align(1),
1829 /*isVolatile=*/false);
1830 Builder.CreateAlignedStore(Val: RawShadowValue,
1831 Ptr: Builder.CreateCall(Callee: NsanGetRawShadowPtr, Args: {Dst}),
1832 Align: Align(1),
1833 /*isVolatile=*/false);
1834
1835 ++NumInstrumentedNonFTMemcpyStores;
1836 return;
1837 }
1838 if (Constant *C; Opts.nsan_propagate_non_ft_const_stores_as_ft &&
1839 (C = dyn_cast<Constant>(Val: StoredValue))) {
1840 // This might be a fp constant stored as an int. Bitcast and store if it has
1841 // appropriate size.
1842 Type *BitcastTy = nullptr; // The FT type to bitcast to.
1843 if (isa<ConstantInt, ConstantDataVector>(Val: C)) {
1844 switch (C->getType()->getScalarSizeInBits()) {
1845 case 32:
1846 BitcastTy = Type::getFloatTy(C&: Context);
1847 break;
1848 case 64:
1849 BitcastTy = Type::getDoubleTy(C&: Context);
1850 break;
1851 case 80:
1852 BitcastTy = Type::getX86_FP80Ty(C&: Context);
1853 break;
1854 default:
1855 break;
1856 }
1857
1858 if (BitcastTy)
1859 if (auto *VectorTy = dyn_cast<VectorType>(Val: C->getType()))
1860 BitcastTy = VectorType::get(ElementType: BitcastTy, EC: VectorTy->getElementCount());
1861 }
1862 if (BitcastTy) {
1863 const MemoryExtents Extents = getMemoryExtentsOrDie(FT: BitcastTy);
1864 Value *ShadowPtr = Builder.CreateCall(
1865 Callee: NsanGetShadowPtrForStore[Extents.ValueType],
1866 Args: {PtrOp, ConstantInt::get(Ty: IntptrTy, V: Extents.NumElts)});
1867 // Bitcast the integer value to the appropriate FT type and extend to 2FT.
1868 Type *ExtVT = Config.getExtendedFPType(FT: BitcastTy);
1869 Value *Shadow =
1870 Builder.CreateFPExt(V: Builder.CreateBitCast(V: C, DestTy: BitcastTy), DestTy: ExtVT);
1871 Builder.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1),
1872 isVolatile: Store.isVolatile());
1873 return;
1874 }
1875 }
1876 // All other stores just reset the shadow value to unknown.
1877 Builder.CreateCall(Callee: NsanSetUnknownFns.getFallback(), Args: {Dst, ValueSize});
1878}
1879
1880void NumericalStabilitySanitizer::propagateShadowValues(
1881 Instruction &Inst, const TargetLibraryInfo &TLI,
1882 const ValueToShadowMap &Map) {
1883 if (auto *Store = dyn_cast<StoreInst>(Val: &Inst)) {
1884 Value *StoredValue = Store->getValueOperand();
1885 Type *VT = StoredValue->getType();
1886 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1887 if (ExtendedVT == nullptr)
1888 return propagateNonFTStore(Store&: *Store, VT, Map);
1889 return propagateFTStore(Store&: *Store, VT, ExtendedVT, Map);
1890 }
1891
1892 if (auto *FCmp = dyn_cast<FCmpInst>(Val: &Inst)) {
1893 emitFCmpCheck(FCmp&: *FCmp, Map);
1894 return;
1895 }
1896
1897 if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
1898 maybeAddSuffixForNsanInterface(CI: CB);
1899 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
1900 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
1901 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
1902 instrumentMemIntrinsic(MI);
1903 return;
1904 }
1905 populateShadowStack(CI&: *CB, TLI, Map);
1906 return;
1907 }
1908
1909 if (auto *RetInst = dyn_cast<ReturnInst>(Val: &Inst)) {
1910 if (!Opts.nsan_check_ret)
1911 return;
1912
1913 Value *RV = RetInst->getReturnValue();
1914 if (RV == nullptr)
1915 return; // This is a `ret void`.
1916 Type *VT = RV->getType();
1917 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1918 if (ExtendedVT == nullptr)
1919 return; // Not an FT ret.
1920 Value *RVShadow = Map.getShadow(V: RV);
1921 IRBuilder<> Builder(RetInst);
1922
1923 RVShadow = emitCheck(V: RV, ShadowV: RVShadow, Builder, Loc: CheckLoc::makeRet());
1924 ++NumInstrumentedFTRets;
1925 // Store tag.
1926 Value *FnAddr =
1927 Builder.CreatePtrToInt(V: Inst.getParent()->getParent(), DestTy: IntptrTy);
1928 Builder.CreateStore(Val: FnAddr, Ptr: NsanShadowRetTag);
1929 // Store value.
1930 Value *ShadowRetValPtr =
1931 Builder.CreateConstGEP2_64(Ty: NsanShadowRetType, Ptr: NsanShadowRetPtr, Idx0: 0, Idx1: 0);
1932 Builder.CreateStore(Val: RVShadow, Ptr: ShadowRetValPtr);
1933 return;
1934 }
1935
1936 if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(Val: &Inst)) {
1937 Value *V = Insert->getOperand(i_nocapture: 1);
1938 Type *VT = V->getType();
1939 Type *ExtendedVT = Config.getExtendedFPType(FT: VT);
1940 if (ExtendedVT == nullptr)
1941 return;
1942 IRBuilder<> Builder(Insert);
1943 emitCheck(V, ShadowV: Map.getShadow(V), Builder, Loc: CheckLoc::makeInsert());
1944 return;
1945 }
1946}
1947
1948// Moves fast math flags from the function to individual instructions, and
1949// removes the attribute from the function.
1950// TODO: Make this controllable with a flag.
1951static void moveFastMathFlags(Function &F,
1952 std::vector<Instruction *> &Instructions) {
1953 FastMathFlags FMF;
1954#define MOVE_FLAG(attr, setter) \
1955 if (F.getFnAttribute(attr).getValueAsString() == "true") { \
1956 F.removeFnAttr(attr); \
1957 FMF.set##setter(); \
1958 }
1959 MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)
1960#undef MOVE_FLAG
1961
1962 for (Instruction *I : Instructions)
1963 if (isa<FPMathOperator>(Val: I))
1964 I->setFastMathFlags(FMF);
1965}
1966
1967bool NumericalStabilitySanitizer::sanitizeFunction(
1968 Function &F, const TargetLibraryInfo &TLI) {
1969 if (!F.hasFnAttribute(Kind: Attribute::SanitizeNumericalStability) ||
1970 F.isDeclaration())
1971 return false;
1972
1973 // This is required to prevent instrumenting call to __nsan_init from within
1974 // the module constructor.
1975 if (F.getName() == kNsanModuleCtorName)
1976 return false;
1977
1978 // The instrumentation maintains:
1979 // - for each IR value `v` of floating-point (or vector floating-point) type
1980 // FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.
1981 // double for float and f128 for double).
1982 // - A shadow memory, which stores `s(v)` for any `v` that has been stored,
1983 // along with a shadow memory tag, which stores whether the value in the
1984 // corresponding shadow memory is valid. Note that this might be
1985 // incorrect if a non-instrumented function stores to memory, or if
1986 // memory is stored to through a char pointer.
1987 // - A shadow stack, which holds `s(v)` for any floating-point argument `v`
1988 // of a call to an instrumented function. This allows
1989 // instrumented functions to retrieve the shadow values for their
1990 // arguments.
1991 // Because instrumented functions can be called from non-instrumented
1992 // functions, the stack needs to include a tag so that the instrumented
1993 // function knows whether shadow values are available for their
1994 // parameters (i.e. whether is was called by an instrumented function).
1995 // When shadow arguments are not available, they have to be recreated by
1996 // extending the precision of the non-shadow arguments to the non-shadow
1997 // value. Non-instrumented functions do not modify (or even know about) the
1998 // shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow
1999 // stack tag is __nsan_shadow_args_tag. The tag is any unique identifier
2000 // for the function (we use the address of the function). Both variables
2001 // are thread local.
2002 // Example:
2003 // calls shadow stack tag shadow stack
2004 // =======================================================================
2005 // non_instrumented_1() 0 0
2006 // |
2007 // v
2008 // instrumented_2(float a) 0 0
2009 // |
2010 // v
2011 // instrumented_3(float b, double c) &instrumented_3 s(b),s(c)
2012 // |
2013 // v
2014 // instrumented_4(float d) &instrumented_4 s(d)
2015 // |
2016 // v
2017 // non_instrumented_5(float e) &non_instrumented_5 s(e)
2018 // |
2019 // v
2020 // instrumented_6(float f) &non_instrumented_5 s(e)
2021 //
2022 // On entry, instrumented_2 checks whether the tag corresponds to its
2023 // function ptr.
2024 // Note that functions reset the tag to 0 after reading shadow parameters.
2025 // This ensures that the function does not erroneously read invalid data if
2026 // called twice in the same stack, once from an instrumented function and
2027 // once from an uninstrumented one. For example, in the following example,
2028 // resetting the tag in (A) ensures that (B) does not reuse the same the
2029 // shadow arguments (which would be incorrect).
2030 // instrumented_1(float a)
2031 // |
2032 // v
2033 // instrumented_2(float b) (A)
2034 // |
2035 // v
2036 // non_instrumented_3()
2037 // |
2038 // v
2039 // instrumented_2(float b) (B)
2040 //
2041 // - A shadow return slot. Any function that returns a floating-point value
2042 // places a shadow return value in __nsan_shadow_ret_val. Again, because
2043 // we might be calling non-instrumented functions, this value is guarded
2044 // by __nsan_shadow_ret_tag marker indicating which instrumented function
2045 // placed the value in __nsan_shadow_ret_val, so that the caller can check
2046 // that this corresponds to the callee. Both variables are thread local.
2047 //
2048 // For example, in the following example, the instrumentation in
2049 // `instrumented_1` rejects the shadow return value from `instrumented_3`
2050 // because it is not tagged as expected (`&instrumented_3` instead of
2051 // `non_instrumented_2`):
2052 //
2053 // instrumented_1()
2054 // |
2055 // v
2056 // float non_instrumented_2()
2057 // |
2058 // v
2059 // float instrumented_3()
2060 //
2061 // Calls of known math functions (sin, cos, exp, ...) are duplicated to call
2062 // their overload on the shadow type.
2063
2064 // Collect all instructions before processing, as creating shadow values
2065 // creates new instructions inside the function.
2066 std::vector<Instruction *> OriginalInstructions;
2067 for (BasicBlock &BB : F)
2068 for (Instruction &Inst : BB)
2069 OriginalInstructions.emplace_back(args: &Inst);
2070
2071 moveFastMathFlags(F, Instructions&: OriginalInstructions);
2072 ValueToShadowMap ValueToShadow(Config);
2073
2074 // In the first pass, we create shadow values for all FT function arguments
2075 // and all phis. This ensures that the DFS of the next pass does not have
2076 // any loops.
2077 std::vector<PHINode *> OriginalPhis;
2078 createShadowArguments(F, TLI, Map&: ValueToShadow);
2079 for (Instruction *I : OriginalInstructions) {
2080 if (PHINode *Phi = dyn_cast<PHINode>(Val: I)) {
2081 if (PHINode *Shadow = maybeCreateShadowPhi(Phi&: *Phi, TLI)) {
2082 OriginalPhis.push_back(x: Phi);
2083 ValueToShadow.setShadow(V&: *Phi, Shadow&: *Shadow);
2084 }
2085 }
2086 }
2087
2088 // Create shadow values for all instructions creating FT values.
2089 for (Instruction *I : OriginalInstructions)
2090 maybeCreateShadowValue(Root&: *I, TLI, Map&: ValueToShadow);
2091
2092 // Propagate shadow values across stores, calls and rets.
2093 for (Instruction *I : OriginalInstructions)
2094 propagateShadowValues(Inst&: *I, TLI, Map: ValueToShadow);
2095
2096 // The last pass populates shadow phis with shadow values.
2097 for (PHINode *Phi : OriginalPhis) {
2098 PHINode *ShadowPhi = cast<PHINode>(Val: ValueToShadow.getShadow(V: Phi));
2099 for (unsigned I : seq(Size: Phi->getNumOperands())) {
2100 Value *V = Phi->getOperand(i_nocapture: I);
2101 Value *Shadow = ValueToShadow.getShadow(V);
2102 BasicBlock *IncomingBB = Phi->getIncomingBlock(i: I);
2103 // For some instructions (e.g. invoke), we create the shadow in a separate
2104 // block, different from the block where the original value is created.
2105 // In that case, the shadow phi might need to refer to this block instead
2106 // of the original block.
2107 // Note that this can only happen for instructions as constant shadows are
2108 // always created in the same block.
2109 ShadowPhi->addIncoming(V: Shadow, BB: IncomingBB);
2110 }
2111 }
2112
2113 return !ValueToShadow.empty();
2114}
2115
2116static uint64_t GetMemOpSize(Value *V) {
2117 uint64_t OpSize = 0;
2118 if (Constant *C = dyn_cast<Constant>(Val: V)) {
2119 auto *CInt = dyn_cast<ConstantInt>(Val: C);
2120 if (CInt && CInt->getValue().getBitWidth() <= 64)
2121 OpSize = CInt->getValue().getZExtValue();
2122 }
2123
2124 return OpSize;
2125}
2126
2127// Instrument the memory intrinsics so that they properly modify the shadow
2128// memory.
2129bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
2130 IRBuilder<> Builder(MI);
2131 if (auto *M = dyn_cast<MemSetInst>(Val: MI)) {
2132 FunctionCallee SetUnknownFn =
2133 NsanSetUnknownFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2134 if (SetUnknownFn.getFunctionType()->getNumParams() == 1)
2135 Builder.CreateCall(Callee: SetUnknownFn, Args: {/*Address=*/M->getArgOperand(i: 0)});
2136 else
2137 Builder.CreateCall(Callee: SetUnknownFn,
2138 Args: {/*Address=*/M->getArgOperand(i: 0),
2139 /*Size=*/Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2140 DestTy: IntptrTy, isSigned: false)});
2141
2142 } else if (auto *M = dyn_cast<MemTransferInst>(Val: MI)) {
2143 FunctionCallee CopyFn =
2144 NsanCopyFns.getFunctionFor(MemOpSize: GetMemOpSize(V: M->getArgOperand(i: 2)));
2145
2146 if (CopyFn.getFunctionType()->getNumParams() == 2)
2147 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2148 /*Source=*/M->getArgOperand(i: 1)});
2149 else
2150 Builder.CreateCall(Callee: CopyFn, Args: {/*Destination=*/M->getArgOperand(i: 0),
2151 /*Source=*/M->getArgOperand(i: 1),
2152 /*Size=*/
2153 Builder.CreateIntCast(V: M->getArgOperand(i: 2),
2154 DestTy: IntptrTy, isSigned: false)});
2155 }
2156 return false;
2157}
2158
2159void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {
2160 Function *Fn = CI->getCalledFunction();
2161 if (Fn == nullptr)
2162 return;
2163
2164 if (!Fn->getName().starts_with(Prefix: "__nsan_"))
2165 return;
2166
2167 if (Fn->getName() == "__nsan_dump_shadow_mem") {
2168 assert(CI->arg_size() == 4 &&
2169 "invalid prototype for __nsan_dump_shadow_mem");
2170 // __nsan_dump_shadow_mem requires an extra parameter with the dynamic
2171 // configuration:
2172 // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)
2173 // | shadow_type_id_for_double
2174 const uint64_t shadow_value_type_ids =
2175 (static_cast<size_t>(Config.byValueType(VT: kLongDouble).getNsanTypeId())
2176 << 16) |
2177 (static_cast<size_t>(Config.byValueType(VT: kDouble).getNsanTypeId())
2178 << 8) |
2179 static_cast<size_t>(Config.byValueType(VT: kFloat).getNsanTypeId());
2180 CI->setArgOperand(i: 3, v: ConstantInt::get(Ty: IntptrTy, V: shadow_value_type_ids));
2181 }
2182}
2183