1//===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of DataFlowSanitizer, a generalised dynamic data flow
11/// analysis.
12///
13/// Unlike other Sanitizer tools, this tool is not designed to detect a specific
14/// class of bugs on its own. Instead, it provides a generic dynamic data flow
15/// analysis framework to be used by clients to help detect application-specific
16/// issues within their own code.
17///
18/// The analysis is based on automatic propagation of data flow labels (also
19/// known as taint labels) through a program as it performs computation.
20///
21/// Argument and return value labels are passed through TLS variables
22/// __dfsan_arg_tls and __dfsan_retval_tls.
23///
24/// Each byte of application memory is backed by a shadow memory byte. The
25/// shadow byte can represent up to 8 labels. On Linux/x86_64, memory is then
26/// laid out as follows:
27///
28/// +--------------------+ 0x800000000000 (top of memory)
29/// | application 3 |
30/// +--------------------+ 0x700000000000
31/// | invalid |
32/// +--------------------+ 0x610000000000
33/// | origin 1 |
34/// +--------------------+ 0x600000000000
35/// | application 2 |
36/// +--------------------+ 0x510000000000
37/// | shadow 1 |
38/// +--------------------+ 0x500000000000
39/// | invalid |
40/// +--------------------+ 0x400000000000
41/// | origin 3 |
42/// +--------------------+ 0x300000000000
43/// | shadow 3 |
44/// +--------------------+ 0x200000000000
45/// | origin 2 |
46/// +--------------------+ 0x110000000000
47/// | invalid |
48/// +--------------------+ 0x100000000000
49/// | shadow 2 |
50/// +--------------------+ 0x010000000000
51/// | application 1 |
52/// +--------------------+ 0x000000000000
53///
54/// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
55/// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
56///
57/// For more information, please refer to the design document:
58/// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html
59//
60//===----------------------------------------------------------------------===//
61
62#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
63#include "InstrumentationOptions.h"
64#include "llvm/ADT/DenseMap.h"
65#include "llvm/ADT/DenseSet.h"
66#include "llvm/ADT/DepthFirstIterator.h"
67#include "llvm/ADT/SmallPtrSet.h"
68#include "llvm/ADT/SmallVector.h"
69#include "llvm/ADT/StringRef.h"
70#include "llvm/ADT/StringSet.h"
71#include "llvm/ADT/iterator.h"
72#include "llvm/Analysis/DomTreeUpdater.h"
73#include "llvm/Analysis/GlobalsModRef.h"
74#include "llvm/Analysis/TargetLibraryInfo.h"
75#include "llvm/Analysis/ValueTracking.h"
76#include "llvm/IR/Argument.h"
77#include "llvm/IR/AttributeMask.h"
78#include "llvm/IR/Attributes.h"
79#include "llvm/IR/BasicBlock.h"
80#include "llvm/IR/Constant.h"
81#include "llvm/IR/Constants.h"
82#include "llvm/IR/DataLayout.h"
83#include "llvm/IR/DerivedTypes.h"
84#include "llvm/IR/Dominators.h"
85#include "llvm/IR/Function.h"
86#include "llvm/IR/GlobalAlias.h"
87#include "llvm/IR/GlobalValue.h"
88#include "llvm/IR/GlobalVariable.h"
89#include "llvm/IR/IRBuilder.h"
90#include "llvm/IR/InstVisitor.h"
91#include "llvm/IR/InstrTypes.h"
92#include "llvm/IR/Instruction.h"
93#include "llvm/IR/Instructions.h"
94#include "llvm/IR/IntrinsicInst.h"
95#include "llvm/IR/MDBuilder.h"
96#include "llvm/IR/Module.h"
97#include "llvm/IR/PassManager.h"
98#include "llvm/IR/Type.h"
99#include "llvm/IR/User.h"
100#include "llvm/IR/Value.h"
101#include "llvm/Support/Alignment.h"
102#include "llvm/Support/Casting.h"
103#include "llvm/Support/ErrorHandling.h"
104#include "llvm/Support/SpecialCaseList.h"
105#include "llvm/Support/VirtualFileSystem.h"
106#include "llvm/TargetParser/Triple.h"
107#include "llvm/Transforms/Utils/BasicBlockUtils.h"
108#include "llvm/Transforms/Utils/Instrumentation.h"
109#include "llvm/Transforms/Utils/Local.h"
110#include <algorithm>
111#include <cassert>
112#include <cstddef>
113#include <cstdint>
114#include <memory>
115#include <set>
116#include <string>
117#include <utility>
118#include <vector>
119
120using namespace llvm;
121
122// This must be consistent with ShadowWidthBits.
123static const Align ShadowTLSAlignment = Align(2);
124
125static const Align MinOriginAlignment = Align(4);
126
127// The size of TLS variables. These constants must be kept in sync with the ones
128// in dfsan.cpp.
129static const unsigned ArgTLSSize = 800;
130static const unsigned RetvalTLSSize = 800;
131
132// The ABI list files control how shadow parameters are passed. The pass treats
133// every function labelled "uninstrumented" in the ABI list file as conforming
134// to the "native" (i.e. unsanitized) ABI. Unless the ABI list contains
135// additional annotations for those functions, a call to one of those functions
136// will produce a warning message, as the labelling behaviour of the function is
137// unknown. The other supported annotations for uninstrumented functions are
138// "functional" and "discard", which are described below under
139// DataFlowSanitizer::WrapperKind.
140// Functions will often be labelled with both "uninstrumented" and one of
141// "functional" or "discard". This will leave the function unchanged by this
142// pass, and create a wrapper function that will call the original.
143//
144// Instrumented functions can also be annotated as "force_zero_labels", which
145// will make all shadow and return values set zero labels.
146// Functions should never be labelled with both "force_zero_labels" and
147// "uninstrumented" or any of the unistrumented wrapper kinds.
148
149static StringRef getGlobalTypeString(const GlobalValue &G) {
150 // Types of GlobalVariables are always pointer types.
151 Type *GType = G.getValueType();
152 // For now we support excluding struct types only.
153 if (StructType *SGType = dyn_cast<StructType>(Val: GType)) {
154 if (!SGType->isLiteral())
155 return SGType->getName();
156 }
157 return "<unknown type>";
158}
159
160namespace {
161
162// Memory map parameters used in application-to-shadow address calculation.
163// Offset = (Addr & ~AndMask) ^ XorMask
164// Shadow = ShadowBase + Offset
165// Origin = (OriginBase + Offset) & ~3ULL
166struct MemoryMapParams {
167 uint64_t AndMask;
168 uint64_t XorMask;
169 uint64_t ShadowBase;
170 uint64_t OriginBase;
171};
172
173} // end anonymous namespace
174
175// NOLINTBEGIN(readability-identifier-naming)
176// aarch64 Linux
177const MemoryMapParams Linux_AArch64_MemoryMapParams = {
178 .AndMask: 0, // AndMask (not used)
179 .XorMask: 0x0B00000000000, // XorMask
180 .ShadowBase: 0, // ShadowBase (not used)
181 .OriginBase: 0x0200000000000, // OriginBase
182};
183
184// x86_64 Linux
185const MemoryMapParams Linux_X86_64_MemoryMapParams = {
186 .AndMask: 0, // AndMask (not used)
187 .XorMask: 0x500000000000, // XorMask
188 .ShadowBase: 0, // ShadowBase (not used)
189 .OriginBase: 0x100000000000, // OriginBase
190};
191// NOLINTEND(readability-identifier-naming)
192
193// loongarch64 Linux
194const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
195 .AndMask: 0, // AndMask (not used)
196 .XorMask: 0x500000000000, // XorMask
197 .ShadowBase: 0, // ShadowBase (not used)
198 .OriginBase: 0x100000000000, // OriginBase
199};
200
201// s390x Linux
202const MemoryMapParams Linux_S390X_MemoryMapParams = {
203 .AndMask: 0xC00000000000, // AndMask
204 .XorMask: 0, // XorMask (not used)
205 .ShadowBase: 0x080000000000, // ShadowBase
206 .OriginBase: 0x1C0000000000, // OriginBase
207};
208
209namespace {
210
211class DFSanABIList {
212 std::unique_ptr<SpecialCaseList> SCL;
213
214public:
215 DFSanABIList() = default;
216
217 void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
218
219 /// Returns whether either this function or its source file are listed in the
220 /// given category.
221 bool isIn(const Function &F, StringRef Category) const {
222 return isIn(M: *F.getParent(), Category) ||
223 SCL->inSection(Section: "dataflow", Prefix: "fun", Query: F.getName(), Category);
224 }
225
226 /// Returns whether this global alias is listed in the given category.
227 ///
228 /// If GA aliases a function, the alias's name is matched as a function name
229 /// would be. Similarly, aliases of globals are matched like globals.
230 bool isIn(const GlobalAlias &GA, StringRef Category) const {
231 if (isIn(M: *GA.getParent(), Category))
232 return true;
233
234 if (isa<FunctionType>(Val: GA.getValueType()))
235 return SCL->inSection(Section: "dataflow", Prefix: "fun", Query: GA.getName(), Category);
236
237 return SCL->inSection(Section: "dataflow", Prefix: "global", Query: GA.getName(), Category) ||
238 SCL->inSection(Section: "dataflow", Prefix: "type", Query: getGlobalTypeString(G: GA),
239 Category);
240 }
241
242 /// Returns whether this module is listed in the given category.
243 bool isIn(const Module &M, StringRef Category) const {
244 return SCL->inSection(Section: "dataflow", Prefix: "src", Query: M.getModuleIdentifier(), Category);
245 }
246};
247
248/// TransformedFunction is used to express the result of transforming one
249/// function type into another. This struct is immutable. It holds metadata
250/// useful for updating calls of the old function to the new type.
251struct TransformedFunction {
252 TransformedFunction(FunctionType *OriginalType, FunctionType *TransformedType,
253 const std::vector<unsigned> &ArgumentIndexMapping,
254 AttributeList &NewParamAttrs)
255 : OriginalType(OriginalType), TransformedType(TransformedType),
256 ArgumentIndexMapping(ArgumentIndexMapping),
257 NewParamAttrs(NewParamAttrs) {}
258
259 // Disallow copies.
260 TransformedFunction(const TransformedFunction &) = delete;
261 TransformedFunction &operator=(const TransformedFunction &) = delete;
262
263 // Allow moves.
264 TransformedFunction(TransformedFunction &&) = default;
265 TransformedFunction &operator=(TransformedFunction &&) = default;
266
267 /// Type of the function before the transformation.
268 FunctionType *OriginalType;
269
270 /// Type of the function after the transformation.
271 FunctionType *TransformedType;
272
273 /// Transforming a function may change the position of arguments. This
274 /// member records the mapping from each argument's old position to its new
275 /// position. Argument positions are zero-indexed. If the transformation
276 /// from F to F' made the first argument of F into the third argument of F',
277 /// then ArgumentIndexMapping[0] will equal 2.
278 std::vector<unsigned> ArgumentIndexMapping;
279
280 /// The (extension) attributes that new Shadow and Origin parameters in
281 /// TransformedType should have.
282 AttributeList NewParamAttrs;
283};
284
285/// Given function attributes from a call site for the original function,
286/// return function attributes appropriate for a call to the transformed
287/// function.
288AttributeList
289transformFunctionAttributes(const TransformedFunction &TransformedFunction,
290 LLVMContext &Ctx, AttributeList CallSiteAttrs) {
291
292 // Construct a vector of AttributeSet for each function argument.
293 std::vector<llvm::AttributeSet> ArgumentAttributes(
294 TransformedFunction.TransformedType->getNumParams());
295
296 // Copy attributes from the parameter of the original function to the
297 // transformed version. 'ArgumentIndexMapping' holds the mapping from
298 // old argument position to new.
299 for (unsigned I = 0, IE = TransformedFunction.ArgumentIndexMapping.size();
300 I < IE; ++I) {
301 unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[I];
302 ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttrs(ArgNo: I);
303 }
304
305 // Copy annotations on varargs arguments.
306 for (unsigned I = TransformedFunction.OriginalType->getNumParams(),
307 IE = CallSiteAttrs.getNumAttrSets();
308 I < IE; ++I) {
309 ArgumentAttributes.push_back(x: CallSiteAttrs.getParamAttrs(ArgNo: I));
310 }
311
312 return AttributeList::get(C&: Ctx, FnAttrs: CallSiteAttrs.getFnAttrs(),
313 RetAttrs: CallSiteAttrs.getRetAttrs(),
314 ArgAttrs: llvm::ArrayRef(ArgumentAttributes));
315}
316
317class DataFlowSanitizer {
318 friend struct DFSanFunction;
319 friend class DFSanVisitor;
320
321 enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
322
323 enum { OriginWidthBits = 32, OriginWidthBytes = OriginWidthBits / 8 };
324
325 /// How should calls to uninstrumented functions be handled?
326 enum WrapperKind {
327 /// This function is present in an uninstrumented form but we don't know
328 /// how it should be handled. Print a warning and call the function anyway.
329 /// Don't label the return value.
330 WK_Warning,
331
332 /// This function does not write to (user-accessible) memory, and its return
333 /// value is unlabelled.
334 WK_Discard,
335
336 /// This function does not write to (user-accessible) memory, and the label
337 /// of its return value is the union of the label of its arguments.
338 WK_Functional,
339
340 /// Instead of calling the function, a custom wrapper __dfsw_F is called,
341 /// where F is the name of the function. This function may wrap the
342 /// original function or provide its own implementation. WK_Custom uses an
343 /// extra pointer argument to return the shadow. This allows the wrapped
344 /// form of the function type to be expressed in C.
345 WK_Custom
346 };
347
348 const InstrumentationOptions &Opts;
349 Module *Mod;
350 LLVMContext *Ctx;
351 Type *Int8Ptr;
352 IntegerType *OriginTy;
353 PointerType *OriginPtrTy;
354 ConstantInt *ZeroOrigin;
355 /// The shadow type for all primitive types and vector types.
356 IntegerType *PrimitiveShadowTy;
357 PointerType *PrimitiveShadowPtrTy;
358 IntegerType *IntptrTy;
359 ConstantInt *ZeroPrimitiveShadow;
360 Constant *ArgTLS;
361 ArrayType *ArgOriginTLSTy;
362 Constant *ArgOriginTLS;
363 Constant *RetvalTLS;
364 Constant *RetvalOriginTLS;
365 FunctionType *DFSanUnionLoadFnTy;
366 FunctionType *DFSanLoadLabelAndOriginFnTy;
367 FunctionType *DFSanUnimplementedFnTy;
368 FunctionType *DFSanWrapperExternWeakNullFnTy;
369 FunctionType *DFSanSetLabelFnTy;
370 FunctionType *DFSanNonzeroLabelFnTy;
371 FunctionType *DFSanVarargWrapperFnTy;
372 FunctionType *DFSanConditionalCallbackFnTy;
373 FunctionType *DFSanConditionalCallbackOriginFnTy;
374 FunctionType *DFSanReachesFunctionCallbackFnTy;
375 FunctionType *DFSanReachesFunctionCallbackOriginFnTy;
376 FunctionType *DFSanCmpCallbackFnTy;
377 FunctionType *DFSanLoadStoreCallbackFnTy;
378 FunctionType *DFSanMemTransferCallbackFnTy;
379 FunctionType *DFSanChainOriginFnTy;
380 FunctionType *DFSanChainOriginIfTaintedFnTy;
381 FunctionType *DFSanMemOriginTransferFnTy;
382 FunctionType *DFSanMemShadowOriginTransferFnTy;
383 FunctionType *DFSanMemShadowOriginConditionalExchangeFnTy;
384 FunctionType *DFSanMaybeStoreOriginFnTy;
385 FunctionCallee DFSanUnionLoadFn;
386 FunctionCallee DFSanLoadLabelAndOriginFn;
387 FunctionCallee DFSanUnimplementedFn;
388 FunctionCallee DFSanWrapperExternWeakNullFn;
389 FunctionCallee DFSanSetLabelFn;
390 FunctionCallee DFSanNonzeroLabelFn;
391 FunctionCallee DFSanVarargWrapperFn;
392 FunctionCallee DFSanLoadCallbackFn;
393 FunctionCallee DFSanStoreCallbackFn;
394 FunctionCallee DFSanMemTransferCallbackFn;
395 FunctionCallee DFSanConditionalCallbackFn;
396 FunctionCallee DFSanConditionalCallbackOriginFn;
397 FunctionCallee DFSanReachesFunctionCallbackFn;
398 FunctionCallee DFSanReachesFunctionCallbackOriginFn;
399 FunctionCallee DFSanCmpCallbackFn;
400 FunctionCallee DFSanChainOriginFn;
401 FunctionCallee DFSanChainOriginIfTaintedFn;
402 FunctionCallee DFSanMemOriginTransferFn;
403 FunctionCallee DFSanMemShadowOriginTransferFn;
404 FunctionCallee DFSanMemShadowOriginConditionalExchangeFn;
405 FunctionCallee DFSanMaybeStoreOriginFn;
406 SmallPtrSet<Value *, 16> DFSanRuntimeFunctions;
407 MDNode *ColdCallWeights;
408 MDNode *OriginStoreWeights;
409 DFSanABIList ABIList;
410 DenseMap<Value *, Function *> UnwrappedFnMap;
411 AttributeMask ReadOnlyNoneAttrs;
412 StringSet<> CombineTaintLookupTableNames;
413
414 /// Memory map parameters used in calculation mapping application addresses
415 /// to shadow addresses and origin addresses.
416 const MemoryMapParams *MapParams;
417
418 Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB);
419 Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos);
420 Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos,
421 Value *ShadowOffset);
422 std::pair<Value *, Value *> getShadowOriginAddress(Value *Addr,
423 Align InstAlignment,
424 BasicBlock::iterator Pos);
425 bool isInstrumented(const Function *F);
426 bool isInstrumented(const GlobalAlias *GA);
427 bool isForceZeroLabels(const Function *F);
428 TransformedFunction getCustomFunctionType(FunctionType *T,
429 TargetLibraryInfo &TLI);
430 WrapperKind getWrapperKind(Function *F);
431 void addGlobalNameSuffix(GlobalValue *GV);
432 void buildExternWeakCheckIfNeeded(IRBuilder<> &IRB, Function *F);
433 Function *buildWrapperFunction(Function *F, StringRef NewFName,
434 GlobalValue::LinkageTypes NewFLink,
435 FunctionType *NewFT);
436 void initializeCallbackFunctions(Module &M);
437 void initializeRuntimeFunctions(Module &M);
438 bool initializeModule(Module &M);
439
440 /// Advances \p OriginAddr to point to the next 32-bit origin and then loads
441 /// from it. Returns the origin's loaded value.
442 Value *loadNextOrigin(BasicBlock::iterator Pos, Align OriginAlign,
443 Value **OriginAddr);
444
445 /// Returns whether the given load byte size is amenable to inlined
446 /// optimization patterns.
447 bool hasLoadSizeForFastPath(uint64_t Size);
448
449 /// Returns whether the pass tracks origins. Supports only TLS ABI mode.
450 bool shouldTrackOrigins();
451
452 /// Returns a zero constant with the shadow type of OrigTy.
453 ///
454 /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
455 /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
456 /// getZeroShadow(other type) = i16(0)
457 Constant *getZeroShadow(Type *OrigTy);
458 /// Returns a zero constant with the shadow type of V's type.
459 Constant *getZeroShadow(Value *V);
460
461 /// Checks if V is a zero shadow.
462 bool isZeroShadow(Value *V);
463
464 /// Returns the shadow type of OrigTy.
465 ///
466 /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
467 /// getShadowTy([n x T]) = [n x getShadowTy(T)]
468 /// getShadowTy(other type) = i16
469 Type *getShadowTy(Type *OrigTy);
470 /// Returns the shadow type of V's type.
471 Type *getShadowTy(Value *V);
472
473 const uint64_t NumOfElementsInArgOrgTLS = ArgTLSSize / OriginWidthBytes;
474
475public:
476 DataFlowSanitizer(const InstrumentationOptions &Opts,
477 const std::vector<std::string> &ABIListFiles,
478 IntrusiveRefCntPtr<vfs::FileSystem> FS);
479
480 bool runImpl(Module &M,
481 llvm::function_ref<TargetLibraryInfo &(Function &)> GetTLI);
482};
483
484struct DFSanFunction {
485 const InstrumentationOptions &Opts;
486 DataFlowSanitizer &DFS;
487 Function *F;
488 DominatorTree DT;
489 bool IsNativeABI;
490 bool IsForceZeroLabels;
491 TargetLibraryInfo &TLI;
492 AllocaInst *LabelReturnAlloca = nullptr;
493 AllocaInst *OriginReturnAlloca = nullptr;
494 DenseMap<Value *, Value *> ValShadowMap;
495 DenseMap<Value *, Value *> ValOriginMap;
496 DenseMap<AllocaInst *, AllocaInst *> AllocaShadowMap;
497 DenseMap<AllocaInst *, AllocaInst *> AllocaOriginMap;
498
499 struct PHIFixupElement {
500 PHINode *Phi;
501 PHINode *ShadowPhi;
502 PHINode *OriginPhi;
503 };
504 std::vector<PHIFixupElement> PHIFixups;
505
506 DenseSet<Instruction *> SkipInsts;
507 std::vector<Value *> NonZeroChecks;
508
509 struct CachedShadow {
510 BasicBlock *Block; // The block where Shadow is defined.
511 Value *Shadow;
512 };
513 /// Maps a value to its latest shadow value in terms of domination tree.
514 DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
515 /// Maps a value to its latest collapsed shadow value it was converted to in
516 /// terms of domination tree. When -dfsan-debug-nonzero-labels is on, this
517 /// cache is used at a post process where CFG blocks are split. So it does not
518 /// cache BasicBlock like CachedShadows, but uses domination between values.
519 DenseMap<Value *, Value *> CachedCollapsedShadows;
520 DenseMap<Value *, std::set<Value *>> ShadowElements;
521
522 DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI,
523 bool IsForceZeroLabels, TargetLibraryInfo &TLI)
524 : Opts(DFS.Opts), DFS(DFS), F(F), IsNativeABI(IsNativeABI),
525 IsForceZeroLabels(IsForceZeroLabels), TLI(TLI) {
526 DT.recalculate(Func&: *F);
527 }
528
529 /// Computes the shadow address for a given function argument.
530 ///
531 /// Shadow = ArgTLS+ArgOffset.
532 Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
533
534 /// Computes the shadow address for a return value.
535 Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
536
537 /// Computes the origin address for a given function argument.
538 ///
539 /// Origin = ArgOriginTLS[ArgNo].
540 Value *getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB);
541
542 /// Computes the origin address for a return value.
543 Value *getRetvalOriginTLS();
544
545 Value *getOrigin(Value *V);
546 void setOrigin(Instruction *I, Value *Origin);
547 /// Generates IR to compute the origin of the last operand with a taint label.
548 Value *combineOperandOrigins(Instruction *Inst);
549 /// Before the instruction Pos, generates IR to compute the last origin with a
550 /// taint label. Labels and origins are from vectors Shadows and Origins
551 /// correspondingly. The generated IR is like
552 /// Sn-1 != Zero ? On-1: ... S2 != Zero ? O2: S1 != Zero ? O1: O0
553 /// When Zero is nullptr, it uses ZeroPrimitiveShadow. Otherwise it can be
554 /// zeros with other bitwidths.
555 Value *combineOrigins(const std::vector<Value *> &Shadows,
556 const std::vector<Value *> &Origins,
557 BasicBlock::iterator Pos, ConstantInt *Zero = nullptr);
558
559 Value *getShadow(Value *V);
560 void setShadow(Instruction *I, Value *Shadow);
561 /// Generates IR to compute the union of the two given shadows, inserting it
562 /// before Pos. The combined value is with primitive type.
563 Value *combineShadows(Value *V1, Value *V2, BasicBlock::iterator Pos);
564 /// Combines the shadow values of V1 and V2, then converts the combined value
565 /// with primitive type into a shadow value with the original type T.
566 Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
567 BasicBlock::iterator Pos);
568 Value *combineOperandShadows(Instruction *Inst);
569
570 /// Generates IR to load shadow and origin corresponding to bytes [\p
571 /// Addr, \p Addr + \p Size), where addr has alignment \p
572 /// InstAlignment, and take the union of each of those shadows. The returned
573 /// shadow always has primitive type.
574 ///
575 /// When tracking loads is enabled, the returned origin is a chain at the
576 /// current stack if the returned shadow is tainted.
577 std::pair<Value *, Value *> loadShadowOrigin(Value *Addr, uint64_t Size,
578 Align InstAlignment,
579 BasicBlock::iterator Pos);
580
581 void storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
582 Align InstAlignment, Value *PrimitiveShadow,
583 Value *Origin, BasicBlock::iterator Pos);
584 /// Applies PrimitiveShadow to all primitive subtypes of T, returning
585 /// the expanded shadow value.
586 ///
587 /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
588 /// EFP([n x T], PS) = [n x EFP(T,PS)]
589 /// EFP(other types, PS) = PS
590 Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
591 BasicBlock::iterator Pos);
592 /// Collapses Shadow into a single primitive shadow value, unioning all
593 /// primitive shadow values in the process. Returns the final primitive
594 /// shadow value.
595 ///
596 /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
597 /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
598 /// CTP(other types, PS) = PS
599 Value *collapseToPrimitiveShadow(Value *Shadow, BasicBlock::iterator Pos);
600
601 void storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign,
602 BasicBlock::iterator Pos);
603
604 Align getShadowAlign(Align InstAlignment);
605
606 // If -dfsan-conditional-callbacks is enabled, insert a callback after a given
607 // branch instruction using the given conditional expression.
608 void addConditionalCallbacksIfEnabled(Instruction &I, Value *Condition);
609
610 // If -dfsan-reaches-function-callbacks is enabled, insert a callback for each
611 // argument and load instruction.
612 void addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB, Instruction &I,
613 Value *Data);
614
615 bool isLookupTableConstant(Value *P);
616
617private:
618 /// Collapses the shadow with aggregate type into a single primitive shadow
619 /// value.
620 template <class AggregateType>
621 Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
622 IRBuilder<> &IRB);
623
624 Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
625
626 /// Returns the shadow value of an argument A.
627 Value *getShadowForTLSArgument(Argument *A);
628
629 /// The fast path of loading shadows.
630 std::pair<Value *, Value *>
631 loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
632 Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
633 BasicBlock::iterator Pos);
634
635 Align getOriginAlign(Align InstAlignment);
636
637 /// Because 4 contiguous bytes share one 4-byte origin, the most accurate load
638 /// is __dfsan_load_label_and_origin. This function returns the union of all
639 /// labels and the origin of the first taint label. However this is an
640 /// additional call with many instructions. To ensure common cases are fast,
641 /// checks if it is possible to load labels and origins without using the
642 /// callback function.
643 ///
644 /// When enabling tracking load instructions, we always use
645 /// __dfsan_load_label_and_origin to reduce code size.
646 bool useCallbackLoadLabelAndOrigin(uint64_t Size, Align InstAlignment);
647
648 /// Returns a chain at the current stack with previous origin V.
649 Value *updateOrigin(Value *V, IRBuilder<> &IRB);
650
651 /// Returns a chain at the current stack with previous origin V if Shadow is
652 /// tainted.
653 Value *updateOriginIfTainted(Value *Shadow, Value *Origin, IRBuilder<> &IRB);
654
655 /// Creates an Intptr = Origin | Origin << 32 if Intptr's size is 64. Returns
656 /// Origin otherwise.
657 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin);
658
659 /// Stores Origin into the address range [StoreOriginAddr, StoreOriginAddr +
660 /// Size).
661 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *StoreOriginAddr,
662 uint64_t StoreOriginSize, Align Alignment);
663
664 /// Stores Origin in terms of its Shadow value.
665 /// * Do not write origins for zero shadows because we do not trace origins
666 /// for untainted sinks.
667 /// * Use __dfsan_maybe_store_origin if there are too many origin store
668 /// instrumentations.
669 void storeOrigin(BasicBlock::iterator Pos, Value *Addr, uint64_t Size,
670 Value *Shadow, Value *Origin, Value *StoreOriginAddr,
671 Align InstAlignment);
672
673 /// Convert a scalar value to an i1 by comparing with 0.
674 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &Name = "");
675
676 bool shouldInstrumentWithCall();
677
678 /// Generates IR to load shadow and origin corresponding to bytes [\p
679 /// Addr, \p Addr + \p Size), where addr has alignment \p
680 /// InstAlignment, and take the union of each of those shadows. The returned
681 /// shadow always has primitive type.
682 std::pair<Value *, Value *>
683 loadShadowOriginSansLoadTracking(Value *Addr, uint64_t Size,
684 Align InstAlignment,
685 BasicBlock::iterator Pos);
686 int NumOriginStores = 0;
687};
688
689class DFSanVisitor : public InstVisitor<DFSanVisitor> {
690public:
691 DFSanFunction &DFSF;
692
693 DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
694
695 const DataLayout &getDataLayout() const {
696 return DFSF.F->getDataLayout();
697 }
698
699 // Combines shadow values and origins for all of I's operands.
700 void visitInstOperands(Instruction &I);
701
702 void visitUnaryOperator(UnaryOperator &UO);
703 void visitBinaryOperator(BinaryOperator &BO);
704 void visitBitCastInst(BitCastInst &BCI);
705 void visitCastInst(CastInst &CI);
706 void visitCmpInst(CmpInst &CI);
707 void visitLandingPadInst(LandingPadInst &LPI);
708 void visitGetElementPtrInst(GetElementPtrInst &GEPI);
709 void visitLoadInst(LoadInst &LI);
710 void visitStoreInst(StoreInst &SI);
711 void visitAtomicRMWInst(AtomicRMWInst &I);
712 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
713 void visitReturnInst(ReturnInst &RI);
714 void visitLibAtomicLoad(CallBase &CB);
715 void visitLibAtomicStore(CallBase &CB);
716 void visitLibAtomicExchange(CallBase &CB);
717 void visitLibAtomicCompareExchange(CallBase &CB);
718 void visitCallBase(CallBase &CB);
719 void visitPHINode(PHINode &PN);
720 void visitExtractElementInst(ExtractElementInst &I);
721 void visitInsertElementInst(InsertElementInst &I);
722 void visitShuffleVectorInst(ShuffleVectorInst &I);
723 void visitExtractValueInst(ExtractValueInst &I);
724 void visitInsertValueInst(InsertValueInst &I);
725 void visitAllocaInst(AllocaInst &I);
726 void visitSelectInst(SelectInst &I);
727 void visitMemSetInst(MemSetInst &I);
728 void visitMemTransferInst(MemTransferInst &I);
729 void visitCondBrInst(CondBrInst &BR);
730 void visitSwitchInst(SwitchInst &SW);
731
732private:
733 void visitCASOrRMW(Align InstAlignment, Instruction &I);
734
735 // Returns false when this is an invoke of a custom function.
736 bool visitWrappedCallBase(Function &F, CallBase &CB);
737
738 // Combines origins for all of I's operands.
739 void visitInstOperandOrigins(Instruction &I);
740
741 void addShadowArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
742 IRBuilder<> &IRB);
743
744 void addOriginArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
745 IRBuilder<> &IRB);
746
747 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB);
748 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB);
749};
750
751bool LibAtomicFunction(const Function &F) {
752 // This is a bit of a hack because TargetLibraryInfo is a function pass.
753 // The DFSan pass would need to be refactored to be function pass oriented
754 // (like MSan is) in order to fit together nicely with TargetLibraryInfo.
755 // We need this check to prevent them from being instrumented, or wrapped.
756 // Match on name and number of arguments.
757 if (!F.hasName() || F.isVarArg())
758 return false;
759 switch (F.arg_size()) {
760 case 4:
761 return F.getName() == "__atomic_load" || F.getName() == "__atomic_store";
762 case 5:
763 return F.getName() == "__atomic_exchange";
764 case 6:
765 return F.getName() == "__atomic_compare_exchange";
766 default:
767 return false;
768 }
769}
770
771} // end anonymous namespace
772
773DataFlowSanitizer::DataFlowSanitizer(
774 const InstrumentationOptions &Opts,
775 const std::vector<std::string> &ABIListFiles,
776 IntrusiveRefCntPtr<vfs::FileSystem> FS)
777 : Opts(Opts) {
778 std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
779 llvm::append_range(C&: AllABIListFiles, R: Opts.dfsan_abilist);
780 ABIList.set(SpecialCaseList::createOrDie(Paths: AllABIListFiles, FS&: *FS));
781
782 CombineTaintLookupTableNames.insert_range(
783 R: Opts.dfsan_combine_taint_lookup_table);
784}
785
786TransformedFunction
787DataFlowSanitizer::getCustomFunctionType(FunctionType *T,
788 TargetLibraryInfo &TLI) {
789 SmallVector<Type *, 4> ArgTypes;
790 AttributeList NewParamAttrs;
791 Attribute::AttrKind ShadowParamExtAttr =
792 TLI.getExtAttrForI8Param(/*Signed=*/false);
793 Attribute::AttrKind OriginParamExtAttr =
794 TLI.getExtAttrForI32Param(/*Signed=*/false);
795
796 // Some parameters of the custom function being constructed are
797 // parameters of T. Record the mapping from parameters of T to
798 // parameters of the custom function, so that parameter attributes
799 // at call sites can be updated.
800 std::vector<unsigned> ArgumentIndexMapping;
801 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
802 Type *ParamType = T->getParamType(i: I);
803 ArgumentIndexMapping.push_back(x: ArgTypes.size());
804 ArgTypes.push_back(Elt: ParamType);
805 }
806 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
807 NewParamAttrs = NewParamAttrs.maybeAddParamAttribute(C&: *Ctx, ArgNo: ArgTypes.size(),
808 Kind: ShadowParamExtAttr);
809 ArgTypes.push_back(Elt: PrimitiveShadowTy);
810 }
811 if (T->isVarArg())
812 ArgTypes.push_back(Elt: PrimitiveShadowPtrTy);
813 Type *RetType = T->getReturnType();
814 if (!RetType->isVoidTy())
815 ArgTypes.push_back(Elt: PrimitiveShadowPtrTy);
816
817 if (shouldTrackOrigins()) {
818 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
819 NewParamAttrs = NewParamAttrs.maybeAddParamAttribute(
820 C&: *Ctx, ArgNo: ArgTypes.size(), Kind: OriginParamExtAttr);
821 ArgTypes.push_back(Elt: OriginTy);
822 }
823 if (T->isVarArg())
824 ArgTypes.push_back(Elt: OriginPtrTy);
825 if (!RetType->isVoidTy())
826 ArgTypes.push_back(Elt: OriginPtrTy);
827 }
828
829 return TransformedFunction(
830 T, FunctionType::get(Result: T->getReturnType(), Params: ArgTypes, isVarArg: T->isVarArg()),
831 ArgumentIndexMapping, NewParamAttrs);
832}
833
834bool DataFlowSanitizer::isZeroShadow(Value *V) {
835 Type *T = V->getType();
836 if (!isa<ArrayType>(Val: T) && !isa<StructType>(Val: T)) {
837 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
838 return CI->isZero();
839 return false;
840 }
841
842 return isa<ConstantAggregateZero>(Val: V);
843}
844
845bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
846 uint64_t ShadowSize = Size * ShadowWidthBytes;
847 return ShadowSize % 8 == 0 || ShadowSize == 4;
848}
849
850bool DataFlowSanitizer::shouldTrackOrigins() {
851 return Opts.dfsan_track_origins;
852}
853
854Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
855 if (!isa<ArrayType>(Val: OrigTy) && !isa<StructType>(Val: OrigTy))
856 return ZeroPrimitiveShadow;
857 Type *ShadowTy = getShadowTy(OrigTy);
858 return ConstantAggregateZero::get(Ty: ShadowTy);
859}
860
861Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
862 return getZeroShadow(OrigTy: V->getType());
863}
864
865static Value *expandFromPrimitiveShadowRecursive(
866 Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
867 Value *PrimitiveShadow, IRBuilder<> &IRB) {
868 if (!isa<ArrayType>(Val: SubShadowTy) && !isa<StructType>(Val: SubShadowTy))
869 return IRB.CreateInsertValue(Agg: Shadow, Val: PrimitiveShadow, Idxs: Indices);
870
871 if (ArrayType *AT = dyn_cast<ArrayType>(Val: SubShadowTy)) {
872 for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
873 Indices.push_back(Elt: Idx);
874 Shadow = expandFromPrimitiveShadowRecursive(
875 Shadow, Indices, SubShadowTy: AT->getElementType(), PrimitiveShadow, IRB);
876 Indices.pop_back();
877 }
878 return Shadow;
879 }
880
881 if (StructType *ST = dyn_cast<StructType>(Val: SubShadowTy)) {
882 for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
883 Indices.push_back(Elt: Idx);
884 Shadow = expandFromPrimitiveShadowRecursive(
885 Shadow, Indices, SubShadowTy: ST->getElementType(N: Idx), PrimitiveShadow, IRB);
886 Indices.pop_back();
887 }
888 return Shadow;
889 }
890 llvm_unreachable("Unexpected shadow type");
891}
892
893bool DFSanFunction::shouldInstrumentWithCall() {
894 return Opts.dfsan_instrument_with_call_threshold >= 0 &&
895 NumOriginStores >= Opts.dfsan_instrument_with_call_threshold;
896}
897
898Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
899 BasicBlock::iterator Pos) {
900 Type *ShadowTy = DFS.getShadowTy(OrigTy: T);
901
902 if (!isa<ArrayType>(Val: ShadowTy) && !isa<StructType>(Val: ShadowTy))
903 return PrimitiveShadow;
904
905 if (DFS.isZeroShadow(V: PrimitiveShadow))
906 return DFS.getZeroShadow(OrigTy: ShadowTy);
907
908 IRBuilder<> IRB(Pos);
909 SmallVector<unsigned, 4> Indices;
910 Value *Shadow = UndefValue::get(T: ShadowTy);
911 Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, SubShadowTy: ShadowTy,
912 PrimitiveShadow, IRB);
913
914 // Caches the primitive shadow value that built the shadow value.
915 CachedCollapsedShadows[Shadow] = PrimitiveShadow;
916 return Shadow;
917}
918
919template <class AggregateType>
920Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
921 IRBuilder<> &IRB) {
922 if (!AT->getNumElements())
923 return DFS.ZeroPrimitiveShadow;
924
925 Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
926 Value *Aggregator = collapseToPrimitiveShadow(Shadow: FirstItem, IRB);
927
928 for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
929 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
930 Value *ShadowInner = collapseToPrimitiveShadow(Shadow: ShadowItem, IRB);
931 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
932 }
933 return Aggregator;
934}
935
936Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
937 IRBuilder<> &IRB) {
938 Type *ShadowTy = Shadow->getType();
939 if (!isa<ArrayType>(Val: ShadowTy) && !isa<StructType>(Val: ShadowTy))
940 return Shadow;
941 if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy))
942 return collapseAggregateShadow<>(AT, Shadow, IRB);
943 if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy))
944 return collapseAggregateShadow<>(AT: ST, Shadow, IRB);
945 llvm_unreachable("Unexpected shadow type");
946}
947
948Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
949 BasicBlock::iterator Pos) {
950 Type *ShadowTy = Shadow->getType();
951 if (!isa<ArrayType>(Val: ShadowTy) && !isa<StructType>(Val: ShadowTy))
952 return Shadow;
953
954 // Checks if the cached collapsed shadow value dominates Pos.
955 Value *&CS = CachedCollapsedShadows[Shadow];
956 if (CS && DT.dominates(Def: CS, User: Pos))
957 return CS;
958
959 IRBuilder<> IRB(Pos);
960 Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
961 // Caches the converted primitive shadow value.
962 CS = PrimitiveShadow;
963 return PrimitiveShadow;
964}
965
966void DFSanFunction::addConditionalCallbacksIfEnabled(Instruction &I,
967 Value *Condition) {
968 if (!Opts.dfsan_conditional_callbacks) {
969 return;
970 }
971 IRBuilder<> IRB(&I);
972 Value *CondShadow = getShadow(V: Condition);
973 CallInst *CI;
974 if (DFS.shouldTrackOrigins()) {
975 Value *CondOrigin = getOrigin(V: Condition);
976 CI = IRB.CreateCall(Callee: DFS.DFSanConditionalCallbackOriginFn,
977 Args: {CondShadow, CondOrigin});
978 CI->maybeAddParamAttr(ArgNo: 1, Kind: TLI.getExtAttrForI32Param(/*Signed=*/false));
979 } else {
980 CI = IRB.CreateCall(Callee: DFS.DFSanConditionalCallbackFn, Args: {CondShadow});
981 }
982 CI->maybeAddParamAttr(ArgNo: 0, Kind: TLI.getExtAttrForI8Param(/*Signed=*/false));
983}
984
985void DFSanFunction::addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB,
986 Instruction &I,
987 Value *Data) {
988 if (!Opts.dfsan_reaches_function_callbacks) {
989 return;
990 }
991 const DebugLoc &dbgloc = I.getDebugLoc();
992 Value *DataShadow = collapseToPrimitiveShadow(Shadow: getShadow(V: Data), IRB);
993 ConstantInt *CILine;
994 llvm::Value *FilePathPtr;
995
996 if (dbgloc.get() == nullptr) {
997 CILine = llvm::ConstantInt::get(Context&: I.getContext(), V: llvm::APInt(32, 0));
998 FilePathPtr = IRB.CreateGlobalString(
999 Str: I.getFunction()->getParent()->getSourceFileName());
1000 } else {
1001 CILine = llvm::ConstantInt::get(Context&: I.getContext(),
1002 V: llvm::APInt(32, dbgloc.getLine()));
1003 FilePathPtr = IRB.CreateGlobalString(Str: dbgloc->getFilename());
1004 }
1005
1006 llvm::Value *FunctionNamePtr =
1007 IRB.CreateGlobalString(Str: I.getFunction()->getName());
1008
1009 CallInst *CB;
1010 std::vector<Value *> args;
1011
1012 Attribute::AttrKind I32ParamExtAttr =
1013 TLI.getExtAttrForI32Param(/*Signed=*/false);
1014 if (DFS.shouldTrackOrigins()) {
1015 Value *DataOrigin = getOrigin(V: Data);
1016 args = { DataShadow, DataOrigin, FilePathPtr, CILine, FunctionNamePtr };
1017 CB = IRB.CreateCall(Callee: DFS.DFSanReachesFunctionCallbackOriginFn, Args: args);
1018 CB->maybeAddParamAttr(ArgNo: 1, Kind: I32ParamExtAttr);
1019 CB->maybeAddParamAttr(ArgNo: 3, Kind: I32ParamExtAttr);
1020 } else {
1021 args = { DataShadow, FilePathPtr, CILine, FunctionNamePtr };
1022 CB = IRB.CreateCall(Callee: DFS.DFSanReachesFunctionCallbackFn, Args: args);
1023 CB->maybeAddParamAttr(ArgNo: 2, Kind: I32ParamExtAttr);
1024 }
1025 CB->maybeAddParamAttr(ArgNo: 0, Kind: TLI.getExtAttrForI8Param(/*Signed=*/false));
1026 CB->setDebugLoc(dbgloc);
1027}
1028
1029Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
1030 if (!OrigTy->isSized())
1031 return PrimitiveShadowTy;
1032 if (isa<IntegerType>(Val: OrigTy))
1033 return PrimitiveShadowTy;
1034 if (isa<VectorType>(Val: OrigTy))
1035 return PrimitiveShadowTy;
1036 if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy))
1037 return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1038 NumElements: AT->getNumElements());
1039 if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1040 SmallVector<Type *, 4> Elements;
1041 for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
1042 Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: I)));
1043 return StructType::get(Context&: *Ctx, Elements);
1044 }
1045 return PrimitiveShadowTy;
1046}
1047
1048Type *DataFlowSanitizer::getShadowTy(Value *V) {
1049 return getShadowTy(OrigTy: V->getType());
1050}
1051
1052bool DataFlowSanitizer::initializeModule(Module &M) {
1053 Triple TargetTriple(M.getTargetTriple());
1054 const DataLayout &DL = M.getDataLayout();
1055
1056 if (TargetTriple.getOS() != Triple::Linux)
1057 report_fatal_error(reason: "unsupported operating system");
1058 switch (TargetTriple.getArch()) {
1059 case Triple::aarch64:
1060 MapParams = &Linux_AArch64_MemoryMapParams;
1061 break;
1062 case Triple::x86_64:
1063 MapParams = &Linux_X86_64_MemoryMapParams;
1064 break;
1065 case Triple::loongarch64:
1066 MapParams = &Linux_LoongArch64_MemoryMapParams;
1067 break;
1068 case Triple::systemz:
1069 MapParams = &Linux_S390X_MemoryMapParams;
1070 break;
1071 default:
1072 report_fatal_error(reason: "unsupported architecture");
1073 }
1074
1075 Mod = &M;
1076 Ctx = &M.getContext();
1077 Int8Ptr = PointerType::getUnqual(C&: *Ctx);
1078 OriginTy = IntegerType::get(C&: *Ctx, NumBits: OriginWidthBits);
1079 OriginPtrTy = PointerType::getUnqual(C&: *Ctx);
1080 PrimitiveShadowTy = IntegerType::get(C&: *Ctx, NumBits: ShadowWidthBits);
1081 PrimitiveShadowPtrTy = PointerType::getUnqual(C&: *Ctx);
1082 IntptrTy = DL.getIntPtrType(C&: *Ctx);
1083 ZeroPrimitiveShadow = ConstantInt::getSigned(Ty: PrimitiveShadowTy, V: 0);
1084 ZeroOrigin = ConstantInt::getSigned(Ty: OriginTy, V: 0);
1085
1086 Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1087 DFSanUnionLoadFnTy = FunctionType::get(Result: PrimitiveShadowTy, Params: DFSanUnionLoadArgs,
1088 /*isVarArg=*/false);
1089 Type *DFSanLoadLabelAndOriginArgs[2] = {Int8Ptr, IntptrTy};
1090 DFSanLoadLabelAndOriginFnTy =
1091 FunctionType::get(Result: IntegerType::get(C&: *Ctx, NumBits: 64), Params: DFSanLoadLabelAndOriginArgs,
1092 /*isVarArg=*/false);
1093 DFSanUnimplementedFnTy = FunctionType::get(
1094 Result: Type::getVoidTy(C&: *Ctx), Params: PointerType::getUnqual(C&: *Ctx), /*isVarArg=*/false);
1095 Type *DFSanWrapperExternWeakNullArgs[2] = {Int8Ptr, Int8Ptr};
1096 DFSanWrapperExternWeakNullFnTy =
1097 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: DFSanWrapperExternWeakNullArgs,
1098 /*isVarArg=*/false);
1099 Type *DFSanSetLabelArgs[4] = {PrimitiveShadowTy, OriginTy,
1100 PointerType::getUnqual(C&: *Ctx), IntptrTy};
1101 DFSanSetLabelFnTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx),
1102 Params: DFSanSetLabelArgs, /*isVarArg=*/false);
1103 DFSanNonzeroLabelFnTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: {},
1104 /*isVarArg=*/false);
1105 DFSanVarargWrapperFnTy = FunctionType::get(
1106 Result: Type::getVoidTy(C&: *Ctx), Params: PointerType::getUnqual(C&: *Ctx), /*isVarArg=*/false);
1107 DFSanConditionalCallbackFnTy =
1108 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: PrimitiveShadowTy,
1109 /*isVarArg=*/false);
1110 Type *DFSanConditionalCallbackOriginArgs[2] = {PrimitiveShadowTy, OriginTy};
1111 DFSanConditionalCallbackOriginFnTy = FunctionType::get(
1112 Result: Type::getVoidTy(C&: *Ctx), Params: DFSanConditionalCallbackOriginArgs,
1113 /*isVarArg=*/false);
1114 Type *DFSanReachesFunctionCallbackArgs[4] = {PrimitiveShadowTy, Int8Ptr,
1115 OriginTy, Int8Ptr};
1116 DFSanReachesFunctionCallbackFnTy =
1117 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: DFSanReachesFunctionCallbackArgs,
1118 /*isVarArg=*/false);
1119 Type *DFSanReachesFunctionCallbackOriginArgs[5] = {
1120 PrimitiveShadowTy, OriginTy, Int8Ptr, OriginTy, Int8Ptr};
1121 DFSanReachesFunctionCallbackOriginFnTy = FunctionType::get(
1122 Result: Type::getVoidTy(C&: *Ctx), Params: DFSanReachesFunctionCallbackOriginArgs,
1123 /*isVarArg=*/false);
1124 DFSanCmpCallbackFnTy =
1125 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: PrimitiveShadowTy,
1126 /*isVarArg=*/false);
1127 DFSanChainOriginFnTy =
1128 FunctionType::get(Result: OriginTy, Params: OriginTy, /*isVarArg=*/false);
1129 Type *DFSanChainOriginIfTaintedArgs[2] = {PrimitiveShadowTy, OriginTy};
1130 DFSanChainOriginIfTaintedFnTy = FunctionType::get(
1131 Result: OriginTy, Params: DFSanChainOriginIfTaintedArgs, /*isVarArg=*/false);
1132 Type *DFSanMaybeStoreOriginArgs[4] = {IntegerType::get(C&: *Ctx, NumBits: ShadowWidthBits),
1133 Int8Ptr, IntptrTy, OriginTy};
1134 DFSanMaybeStoreOriginFnTy = FunctionType::get(
1135 Result: Type::getVoidTy(C&: *Ctx), Params: DFSanMaybeStoreOriginArgs, /*isVarArg=*/false);
1136 Type *DFSanMemOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1137 DFSanMemOriginTransferFnTy = FunctionType::get(
1138 Result: Type::getVoidTy(C&: *Ctx), Params: DFSanMemOriginTransferArgs, /*isVarArg=*/false);
1139 Type *DFSanMemShadowOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1140 DFSanMemShadowOriginTransferFnTy =
1141 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: DFSanMemShadowOriginTransferArgs,
1142 /*isVarArg=*/false);
1143 Type *DFSanMemShadowOriginConditionalExchangeArgs[5] = {
1144 IntegerType::get(C&: *Ctx, NumBits: 8), Int8Ptr, Int8Ptr, Int8Ptr, IntptrTy};
1145 DFSanMemShadowOriginConditionalExchangeFnTy = FunctionType::get(
1146 Result: Type::getVoidTy(C&: *Ctx), Params: DFSanMemShadowOriginConditionalExchangeArgs,
1147 /*isVarArg=*/false);
1148 Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
1149 DFSanLoadStoreCallbackFnTy =
1150 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: DFSanLoadStoreCallbackArgs,
1151 /*isVarArg=*/false);
1152 Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1153 DFSanMemTransferCallbackFnTy =
1154 FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: DFSanMemTransferCallbackArgs,
1155 /*isVarArg=*/false);
1156
1157 ColdCallWeights = MDBuilder(*Ctx).createUnlikelyBranchWeights();
1158 OriginStoreWeights = MDBuilder(*Ctx).createUnlikelyBranchWeights();
1159 return true;
1160}
1161
1162bool DataFlowSanitizer::isInstrumented(const Function *F) {
1163 return !ABIList.isIn(F: *F, Category: "uninstrumented");
1164}
1165
1166bool DataFlowSanitizer::isInstrumented(const GlobalAlias *GA) {
1167 return !ABIList.isIn(GA: *GA, Category: "uninstrumented");
1168}
1169
1170bool DataFlowSanitizer::isForceZeroLabels(const Function *F) {
1171 return ABIList.isIn(F: *F, Category: "force_zero_labels");
1172}
1173
1174DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(Function *F) {
1175 if (ABIList.isIn(F: *F, Category: "functional"))
1176 return WK_Functional;
1177 if (ABIList.isIn(F: *F, Category: "discard"))
1178 return WK_Discard;
1179 if (ABIList.isIn(F: *F, Category: "custom"))
1180 return WK_Custom;
1181
1182 return WK_Warning;
1183}
1184
1185void DataFlowSanitizer::addGlobalNameSuffix(GlobalValue *GV) {
1186 if (!Opts.dfsan_add_global_name_suffix)
1187 return;
1188
1189 std::string GVName = std::string(GV->getName()), Suffix = ".dfsan";
1190 GV->setName(GVName + Suffix);
1191
1192 // Try to change the name of the function in module inline asm. We only do
1193 // this for specific asm directives, currently only ".symver", to try to avoid
1194 // corrupting asm which happens to contain the symbol name as a substring.
1195 // Note that the substitution for .symver assumes that the versioned symbol
1196 // also has an instrumented name.
1197 for (Module::GlobalAsmFragment &Frag :
1198 GV->getParent()->getModuleInlineAsm()) {
1199 std::string SearchStr = ".symver " + GVName + ",";
1200 size_t Pos = Frag.Asm.find(str: SearchStr);
1201 if (Pos != std::string::npos) {
1202 Frag.Asm.replace(pos: Pos, n: SearchStr.size(),
1203 str: ".symver " + GVName + Suffix + ",");
1204 Pos = Frag.Asm.find(c: '@');
1205
1206 if (Pos == std::string::npos)
1207 report_fatal_error(reason: Twine("unsupported .symver: ", Frag.Asm));
1208
1209 Frag.Asm.replace(pos: Pos, n: 1, str: Suffix + "@");
1210 }
1211 }
1212}
1213
1214void DataFlowSanitizer::buildExternWeakCheckIfNeeded(IRBuilder<> &IRB,
1215 Function *F) {
1216 // If the function we are wrapping was ExternWeak, it may be null.
1217 // The original code before calling this wrapper may have checked for null,
1218 // but replacing with a known-to-not-be-null wrapper can break this check.
1219 // When replacing uses of the extern weak function with the wrapper we try
1220 // to avoid replacing uses in conditionals, but this is not perfect.
1221 // In the case where we fail, and accidentally optimize out a null check
1222 // for a extern weak function, add a check here to help identify the issue.
1223 if (GlobalValue::isExternalWeakLinkage(Linkage: F->getLinkage())) {
1224 std::vector<Value *> Args;
1225 Args.push_back(x: F);
1226 Args.push_back(x: IRB.CreateGlobalString(Str: F->getName()));
1227 IRB.CreateCall(Callee: DFSanWrapperExternWeakNullFn, Args);
1228 }
1229}
1230
1231Function *
1232DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
1233 GlobalValue::LinkageTypes NewFLink,
1234 FunctionType *NewFT) {
1235 FunctionType *FT = F->getFunctionType();
1236 Function *NewF = Function::Create(Ty: NewFT, Linkage: NewFLink, AddrSpace: F->getAddressSpace(),
1237 N: NewFName, M: F->getParent());
1238 NewF->copyAttributesFrom(Src: F);
1239 NewF->removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
1240 Ty: NewFT->getReturnType(), AS: NewF->getAttributes().getRetAttrs()));
1241
1242 BasicBlock *BB = BasicBlock::Create(Context&: *Ctx, Name: "entry", Parent: NewF);
1243 if (F->isVarArg()) {
1244 NewF->removeFnAttr(Kind: "split-stack");
1245 CallInst::Create(Func: DFSanVarargWrapperFn,
1246 Args: IRBuilder<>(BB).CreateGlobalString(Str: F->getName()), NameStr: "", InsertBefore: BB);
1247 new UnreachableInst(*Ctx, BB);
1248 } else {
1249 auto ArgIt = pointer_iterator<Argument *>(NewF->arg_begin());
1250 std::vector<Value *> Args(ArgIt, ArgIt + FT->getNumParams());
1251
1252 CallInst *CI = CallInst::Create(Func: F, Args, NameStr: "", InsertBefore: BB);
1253 if (FT->getReturnType()->isVoidTy())
1254 ReturnInst::Create(C&: *Ctx, InsertAtEnd: BB);
1255 else
1256 ReturnInst::Create(C&: *Ctx, retVal: CI, InsertBefore: BB);
1257 }
1258
1259 return NewF;
1260}
1261
1262// Initialize DataFlowSanitizer runtime functions and declare them in the module
1263void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
1264 LLVMContext &C = M.getContext();
1265 Attribute::AttrKind I8ParamExtAttr =
1266 TargetLibraryInfo::getExtAttrForI8Param(/*Signed=*/false);
1267 Attribute::AttrKind I32ParamExtAttr =
1268 TargetLibraryInfo::getExtAttrForI32Param(T: M.getTargetTriple(),
1269 /*Signed=*/false);
1270 {
1271 AttributeList AL;
1272 AL = AL.addFnAttribute(C, Kind: Attribute::NoUnwind);
1273 AL = AL.addFnAttribute(
1274 C, Attr: Attribute::getWithMemoryEffects(Context&: C, ME: MemoryEffects::readOnly()));
1275 AL = AL.addRetAttribute(C, Kind: Attribute::ZExt);
1276 DFSanUnionLoadFn =
1277 Mod->getOrInsertFunction(Name: "__dfsan_union_load", T: DFSanUnionLoadFnTy, AttributeList: AL);
1278 }
1279 {
1280 AttributeList AL;
1281 AL = AL.addFnAttribute(C, Kind: Attribute::NoUnwind);
1282 AL = AL.addFnAttribute(
1283 C, Attr: Attribute::getWithMemoryEffects(Context&: C, ME: MemoryEffects::readOnly()));
1284 AL = AL.addRetAttribute(C, Kind: Attribute::ZExt);
1285 DFSanLoadLabelAndOriginFn = Mod->getOrInsertFunction(
1286 Name: "__dfsan_load_label_and_origin", T: DFSanLoadLabelAndOriginFnTy, AttributeList: AL);
1287 }
1288 DFSanUnimplementedFn =
1289 Mod->getOrInsertFunction(Name: "__dfsan_unimplemented", T: DFSanUnimplementedFnTy);
1290 DFSanWrapperExternWeakNullFn = Mod->getOrInsertFunction(
1291 Name: "__dfsan_wrapper_extern_weak_null", T: DFSanWrapperExternWeakNullFnTy);
1292 {
1293 AttributeList AL;
1294 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1295 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 1, Kind: I32ParamExtAttr);
1296 DFSanSetLabelFn =
1297 Mod->getOrInsertFunction(Name: "__dfsan_set_label", T: DFSanSetLabelFnTy, AttributeList: AL);
1298 }
1299 DFSanNonzeroLabelFn =
1300 Mod->getOrInsertFunction(Name: "__dfsan_nonzero_label", T: DFSanNonzeroLabelFnTy);
1301 DFSanVarargWrapperFn = Mod->getOrInsertFunction(Name: "__dfsan_vararg_wrapper",
1302 T: DFSanVarargWrapperFnTy);
1303 {
1304 AttributeList AL;
1305 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I32ParamExtAttr);
1306 AL = AL.addRetAttribute(C&: M.getContext(), Kind: Attribute::ZExt);
1307 DFSanChainOriginFn = Mod->getOrInsertFunction(Name: "__dfsan_chain_origin",
1308 T: DFSanChainOriginFnTy, AttributeList: AL);
1309 }
1310 {
1311 AttributeList AL;
1312 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1313 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 1, Kind: I32ParamExtAttr);
1314 AL = AL.addRetAttribute(C&: M.getContext(), Kind: Attribute::ZExt);
1315 DFSanChainOriginIfTaintedFn = Mod->getOrInsertFunction(
1316 Name: "__dfsan_chain_origin_if_tainted", T: DFSanChainOriginIfTaintedFnTy, AttributeList: AL);
1317 }
1318 DFSanMemOriginTransferFn = Mod->getOrInsertFunction(
1319 Name: "__dfsan_mem_origin_transfer", T: DFSanMemOriginTransferFnTy);
1320
1321 DFSanMemShadowOriginTransferFn = Mod->getOrInsertFunction(
1322 Name: "__dfsan_mem_shadow_origin_transfer", T: DFSanMemShadowOriginTransferFnTy);
1323
1324 {
1325 AttributeList AL;
1326 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1327 DFSanMemShadowOriginConditionalExchangeFn = Mod->getOrInsertFunction(
1328 Name: "__dfsan_mem_shadow_origin_conditional_exchange",
1329 T: DFSanMemShadowOriginConditionalExchangeFnTy, AttributeList: AL);
1330 }
1331
1332 {
1333 AttributeList AL;
1334 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1335 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 3, Kind: I32ParamExtAttr);
1336 DFSanMaybeStoreOriginFn = Mod->getOrInsertFunction(
1337 Name: "__dfsan_maybe_store_origin", T: DFSanMaybeStoreOriginFnTy, AttributeList: AL);
1338 }
1339
1340 DFSanRuntimeFunctions.insert(
1341 Ptr: DFSanUnionLoadFn.getCallee()->stripPointerCasts());
1342 DFSanRuntimeFunctions.insert(
1343 Ptr: DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
1344 DFSanRuntimeFunctions.insert(
1345 Ptr: DFSanUnimplementedFn.getCallee()->stripPointerCasts());
1346 DFSanRuntimeFunctions.insert(
1347 Ptr: DFSanWrapperExternWeakNullFn.getCallee()->stripPointerCasts());
1348 DFSanRuntimeFunctions.insert(
1349 Ptr: DFSanSetLabelFn.getCallee()->stripPointerCasts());
1350 DFSanRuntimeFunctions.insert(
1351 Ptr: DFSanNonzeroLabelFn.getCallee()->stripPointerCasts());
1352 DFSanRuntimeFunctions.insert(
1353 Ptr: DFSanVarargWrapperFn.getCallee()->stripPointerCasts());
1354 DFSanRuntimeFunctions.insert(
1355 Ptr: DFSanLoadCallbackFn.getCallee()->stripPointerCasts());
1356 DFSanRuntimeFunctions.insert(
1357 Ptr: DFSanStoreCallbackFn.getCallee()->stripPointerCasts());
1358 DFSanRuntimeFunctions.insert(
1359 Ptr: DFSanMemTransferCallbackFn.getCallee()->stripPointerCasts());
1360 DFSanRuntimeFunctions.insert(
1361 Ptr: DFSanConditionalCallbackFn.getCallee()->stripPointerCasts());
1362 DFSanRuntimeFunctions.insert(
1363 Ptr: DFSanConditionalCallbackOriginFn.getCallee()->stripPointerCasts());
1364 DFSanRuntimeFunctions.insert(
1365 Ptr: DFSanReachesFunctionCallbackFn.getCallee()->stripPointerCasts());
1366 DFSanRuntimeFunctions.insert(
1367 Ptr: DFSanReachesFunctionCallbackOriginFn.getCallee()->stripPointerCasts());
1368 DFSanRuntimeFunctions.insert(
1369 Ptr: DFSanCmpCallbackFn.getCallee()->stripPointerCasts());
1370 DFSanRuntimeFunctions.insert(
1371 Ptr: DFSanChainOriginFn.getCallee()->stripPointerCasts());
1372 DFSanRuntimeFunctions.insert(
1373 Ptr: DFSanChainOriginIfTaintedFn.getCallee()->stripPointerCasts());
1374 DFSanRuntimeFunctions.insert(
1375 Ptr: DFSanMemOriginTransferFn.getCallee()->stripPointerCasts());
1376 DFSanRuntimeFunctions.insert(
1377 Ptr: DFSanMemShadowOriginTransferFn.getCallee()->stripPointerCasts());
1378 DFSanRuntimeFunctions.insert(
1379 Ptr: DFSanMemShadowOriginConditionalExchangeFn.getCallee()
1380 ->stripPointerCasts());
1381 DFSanRuntimeFunctions.insert(
1382 Ptr: DFSanMaybeStoreOriginFn.getCallee()->stripPointerCasts());
1383}
1384
1385// Initializes event callback functions and declare them in the module
1386void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
1387 Attribute::AttrKind I8ParamExtAttr =
1388 TargetLibraryInfo::getExtAttrForI8Param(/*Signed=*/false);
1389 Attribute::AttrKind I32ParamExtAttr =
1390 TargetLibraryInfo::getExtAttrForI32Param(T: M.getTargetTriple(),
1391 /*Signed=*/false);
1392 {
1393 AttributeList AL;
1394 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1395 DFSanLoadCallbackFn = Mod->getOrInsertFunction(
1396 Name: "__dfsan_load_callback", T: DFSanLoadStoreCallbackFnTy, AttributeList: AL);
1397 }
1398 {
1399 AttributeList AL;
1400 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1401 DFSanStoreCallbackFn = Mod->getOrInsertFunction(
1402 Name: "__dfsan_store_callback", T: DFSanLoadStoreCallbackFnTy, AttributeList: AL);
1403 }
1404 DFSanMemTransferCallbackFn = Mod->getOrInsertFunction(
1405 Name: "__dfsan_mem_transfer_callback", T: DFSanMemTransferCallbackFnTy);
1406 {
1407 AttributeList AL;
1408 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1409 DFSanCmpCallbackFn = Mod->getOrInsertFunction(Name: "__dfsan_cmp_callback",
1410 T: DFSanCmpCallbackFnTy, AttributeList: AL);
1411 }
1412 {
1413 AttributeList AL;
1414 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1415 DFSanConditionalCallbackFn = Mod->getOrInsertFunction(
1416 Name: "__dfsan_conditional_callback", T: DFSanConditionalCallbackFnTy, AttributeList: AL);
1417 }
1418 {
1419 AttributeList AL;
1420 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1421 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 1, Kind: I32ParamExtAttr);
1422 DFSanConditionalCallbackOriginFn =
1423 Mod->getOrInsertFunction(Name: "__dfsan_conditional_callback_origin",
1424 T: DFSanConditionalCallbackOriginFnTy, AttributeList: AL);
1425 }
1426 {
1427 AttributeList AL;
1428 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1429 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: I32ParamExtAttr);
1430 DFSanReachesFunctionCallbackFn =
1431 Mod->getOrInsertFunction(Name: "__dfsan_reaches_function_callback",
1432 T: DFSanReachesFunctionCallbackFnTy, AttributeList: AL);
1433 }
1434 {
1435 AttributeList AL;
1436 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 0, Kind: I8ParamExtAttr);
1437 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 1, Kind: I32ParamExtAttr);
1438 AL = AL.maybeAddParamAttribute(C&: M.getContext(), ArgNo: 3, Kind: I32ParamExtAttr);
1439 DFSanReachesFunctionCallbackOriginFn =
1440 Mod->getOrInsertFunction(Name: "__dfsan_reaches_function_callback_origin",
1441 T: DFSanReachesFunctionCallbackOriginFnTy, AttributeList: AL);
1442 }
1443}
1444
1445bool DataFlowSanitizer::runImpl(
1446 Module &M, llvm::function_ref<TargetLibraryInfo &(Function &)> GetTLI) {
1447 initializeModule(M);
1448
1449 if (ABIList.isIn(M, Category: "skip"))
1450 return false;
1451
1452 const unsigned InitialGlobalSize = M.global_size();
1453 const unsigned InitialModuleSize = M.size();
1454
1455 bool Changed = false;
1456
1457 auto GetOrInsertGlobal = [this, &Changed](StringRef Name,
1458 Type *Ty) -> Constant * {
1459 GlobalVariable *G = Mod->getOrInsertGlobal(Name, Ty);
1460 Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
1461 G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
1462 return G;
1463 };
1464
1465 // These globals must be kept in sync with the ones in dfsan.cpp.
1466 ArgTLS =
1467 GetOrInsertGlobal("__dfsan_arg_tls",
1468 ArrayType::get(ElementType: Type::getInt64Ty(C&: *Ctx), NumElements: ArgTLSSize / 8));
1469 RetvalTLS = GetOrInsertGlobal(
1470 "__dfsan_retval_tls",
1471 ArrayType::get(ElementType: Type::getInt64Ty(C&: *Ctx), NumElements: RetvalTLSSize / 8));
1472 ArgOriginTLSTy = ArrayType::get(ElementType: OriginTy, NumElements: NumOfElementsInArgOrgTLS);
1473 ArgOriginTLS = GetOrInsertGlobal("__dfsan_arg_origin_tls", ArgOriginTLSTy);
1474 RetvalOriginTLS = GetOrInsertGlobal("__dfsan_retval_origin_tls", OriginTy);
1475
1476 (void)Mod->getOrInsertGlobal(Name: "__dfsan_track_origins", Ty: OriginTy, CreateGlobalCallback: [&] {
1477 Changed = true;
1478 return new GlobalVariable(
1479 M, OriginTy, true, GlobalValue::WeakODRLinkage,
1480 ConstantInt::getSigned(Ty: OriginTy, V: Opts.dfsan_track_origins),
1481 "__dfsan_track_origins");
1482 });
1483
1484 initializeCallbackFunctions(M);
1485 initializeRuntimeFunctions(M);
1486
1487 std::vector<Function *> FnsToInstrument;
1488 SmallPtrSet<Function *, 2> FnsWithNativeABI;
1489 SmallPtrSet<Function *, 2> FnsWithForceZeroLabel;
1490 SmallPtrSet<Constant *, 1> PersonalityFns;
1491 for (Function &F : M)
1492 if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(Ptr: &F) &&
1493 !LibAtomicFunction(F) &&
1494 !F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation)) {
1495 FnsToInstrument.push_back(x: &F);
1496 if (F.hasPersonalityFn())
1497 PersonalityFns.insert(Ptr: F.getPersonalityFn()->stripPointerCasts());
1498 }
1499
1500 if (Opts.dfsan_ignore_personality_routine) {
1501 for (auto *C : PersonalityFns) {
1502 assert(isa<Function>(C) && "Personality routine is not a function!");
1503 Function *F = cast<Function>(Val: C);
1504 if (!isInstrumented(F))
1505 llvm::erase(C&: FnsToInstrument, V: F);
1506 }
1507 }
1508
1509 // Give function aliases prefixes when necessary, and build wrappers where the
1510 // instrumentedness is inconsistent.
1511 for (GlobalAlias &GA : llvm::make_early_inc_range(Range: M.aliases())) {
1512 // Don't stop on weak. We assume people aren't playing games with the
1513 // instrumentedness of overridden weak aliases.
1514 auto *F = dyn_cast<Function>(Val: GA.getAliaseeObject());
1515 if (!F)
1516 continue;
1517
1518 bool GAInst = isInstrumented(GA: &GA), FInst = isInstrumented(F);
1519 if (GAInst && FInst) {
1520 addGlobalNameSuffix(GV: &GA);
1521 } else if (GAInst != FInst) {
1522 // Non-instrumented alias of an instrumented function, or vice versa.
1523 // Replace the alias with a native-ABI wrapper of the aliasee. The pass
1524 // below will take care of instrumenting it.
1525 Function *NewF =
1526 buildWrapperFunction(F, NewFName: "", NewFLink: GA.getLinkage(), NewFT: F->getFunctionType());
1527 GA.replaceAllUsesWith(V: NewF);
1528 NewF->takeName(V: &GA);
1529 GA.eraseFromParent();
1530 FnsToInstrument.push_back(x: NewF);
1531 }
1532 }
1533
1534 // TODO: This could be more precise.
1535 ReadOnlyNoneAttrs.addAttribute(Val: Attribute::Memory);
1536
1537 // First, change the ABI of every function in the module. ABI-listed
1538 // functions keep their original ABI and get a wrapper function.
1539 for (std::vector<Function *>::iterator FI = FnsToInstrument.begin(),
1540 FE = FnsToInstrument.end();
1541 FI != FE; ++FI) {
1542 Function &F = **FI;
1543 FunctionType *FT = F.getFunctionType();
1544
1545 bool IsZeroArgsVoidRet = (FT->getNumParams() == 0 && !FT->isVarArg() &&
1546 FT->getReturnType()->isVoidTy());
1547
1548 if (isInstrumented(F: &F)) {
1549 if (isForceZeroLabels(F: &F))
1550 FnsWithForceZeroLabel.insert(Ptr: &F);
1551
1552 // Instrumented functions get a '.dfsan' suffix. This allows us to more
1553 // easily identify cases of mismatching ABIs. This naming scheme is
1554 // mangling-compatible (see Itanium ABI), using a vendor-specific suffix.
1555 addGlobalNameSuffix(GV: &F);
1556 } else if (!IsZeroArgsVoidRet || getWrapperKind(F: &F) == WK_Custom) {
1557 // Build a wrapper function for F. The wrapper simply calls F, and is
1558 // added to FnsToInstrument so that any instrumentation according to its
1559 // WrapperKind is done in the second pass below.
1560
1561 // If the function being wrapped has local linkage, then preserve the
1562 // function's linkage in the wrapper function.
1563 GlobalValue::LinkageTypes WrapperLinkage =
1564 F.hasLocalLinkage() ? F.getLinkage()
1565 : GlobalValue::LinkOnceODRLinkage;
1566
1567 Function *NewF = buildWrapperFunction(
1568 F: &F,
1569 NewFName: (shouldTrackOrigins() ? std::string("dfso$") : std::string("dfsw$")) +
1570 std::string(F.getName()),
1571 NewFLink: WrapperLinkage, NewFT: FT);
1572 NewF->removeFnAttrs(Attrs: ReadOnlyNoneAttrs);
1573
1574 // Extern weak functions can sometimes be null at execution time.
1575 // Code will sometimes check if an extern weak function is null.
1576 // This could look something like:
1577 // declare extern_weak i8 @my_func(i8)
1578 // %cmp = icmp ne ptr @my_func, null
1579 // br i1 %cmp, label %use_my_func, label %avoid_my_func
1580 // The @"dfsw$my_func" wrapper is never null, so if we replace this use
1581 // in the comparison, the icmp will simplify to false and we have
1582 // accidentally optimized away a null check that is necessary.
1583 // This can lead to a crash when the null extern_weak my_func is called.
1584 //
1585 // To prevent (the most common pattern of) this problem,
1586 // do not replace uses in comparisons with the wrapper.
1587 // We definitely want to replace uses in call instructions.
1588 // Other uses (e.g. store the function address somewhere) might be
1589 // called or compared or both - this case may not be handled correctly.
1590 // We will default to replacing with wrapper in cases we are unsure.
1591 auto IsNotCmpUse = [](Use &U) -> bool {
1592 User *Usr = U.getUser();
1593 if (Instruction *I = dyn_cast<Instruction>(Val: Usr)) {
1594 if (I->getOpcode() == Instruction::ICmp) {
1595 return false;
1596 }
1597 }
1598 return true;
1599 };
1600 F.replaceUsesWithIf(New: NewF, ShouldReplace: IsNotCmpUse);
1601
1602 UnwrappedFnMap[NewF] = &F;
1603 *FI = NewF;
1604
1605 if (!F.isDeclaration()) {
1606 // This function is probably defining an interposition of an
1607 // uninstrumented function and hence needs to keep the original ABI.
1608 // But any functions it may call need to use the instrumented ABI, so
1609 // we instrument it in a mode which preserves the original ABI.
1610 FnsWithNativeABI.insert(Ptr: &F);
1611
1612 // This code needs to rebuild the iterators, as they may be invalidated
1613 // by the push_back, taking care that the new range does not include
1614 // any functions added by this code.
1615 size_t N = FI - FnsToInstrument.begin(),
1616 Count = FE - FnsToInstrument.begin();
1617 FnsToInstrument.push_back(x: &F);
1618 FI = FnsToInstrument.begin() + N;
1619 FE = FnsToInstrument.begin() + Count;
1620 }
1621 // Hopefully, nobody will try to indirectly call a vararg
1622 // function... yet.
1623 } else if (FT->isVarArg()) {
1624 UnwrappedFnMap[&F] = &F;
1625 *FI = nullptr;
1626 }
1627 }
1628
1629 for (Function *F : FnsToInstrument) {
1630 if (!F || F->isDeclaration())
1631 continue;
1632
1633 removeUnreachableBlocks(F&: *F);
1634
1635 DFSanFunction DFSF(*this, F, FnsWithNativeABI.count(Ptr: F),
1636 FnsWithForceZeroLabel.count(Ptr: F), GetTLI(*F));
1637
1638 // Skip wrappers: they may be called from uninstrumented code (e.g. the
1639 // runtime calling the reaches function callback), so their argument
1640 // shadows are stale, which can cause infinite callback recursion.
1641 if (Opts.dfsan_reaches_function_callbacks && !UnwrappedFnMap.count(Val: F)) {
1642 // Instructions inserted below all precede this one.
1643 Instruction *FirstOrigInst = &F->getEntryBlock().front();
1644
1645 // Add callback for arguments reaching this function.
1646 for (auto &FArg : F->args()) {
1647 Instruction *Next = &F->getEntryBlock().front();
1648 Value *FArgShadow = DFSF.getShadow(V: &FArg);
1649 if (isZeroShadow(V: FArgShadow))
1650 continue;
1651 if (Instruction *FArgShadowInst = dyn_cast<Instruction>(Val: FArgShadow)) {
1652 Next = FArgShadowInst->getNextNode();
1653 }
1654 if (shouldTrackOrigins()) {
1655 if (Instruction *Origin =
1656 dyn_cast<Instruction>(Val: DFSF.getOrigin(V: &FArg))) {
1657 // Ensure IRB insertion point is after loads for shadow and origin.
1658 Instruction *OriginNext = Origin->getNextNode();
1659 if (Next->comesBefore(Other: OriginNext)) {
1660 Next = OriginNext;
1661 }
1662 }
1663 }
1664 IRBuilder<> IRB(Next);
1665 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, I&: *Next, Data: &FArg);
1666 }
1667
1668 // Don't instrument the instrumentation inserted above; e.g. the callback
1669 // calls would otherwise clobber __dfsan_arg_tls.
1670 for (Instruction &I : F->getEntryBlock()) {
1671 if (&I == FirstOrigInst)
1672 break;
1673 DFSF.SkipInsts.insert(V: &I);
1674 }
1675 }
1676
1677 // DFSanVisitor may create new basic blocks, which confuses df_iterator.
1678 // Build a copy of the list before iterating over it.
1679 SmallVector<BasicBlock *, 4> BBList(depth_first(G: &F->getEntryBlock()));
1680
1681 for (BasicBlock *BB : BBList) {
1682 Instruction *Inst = &BB->front();
1683 while (true) {
1684 // DFSanVisitor may split the current basic block, changing the current
1685 // instruction's next pointer and moving the next instruction to the
1686 // tail block from which we should continue.
1687 Instruction *Next = Inst->getNextNode();
1688 // DFSanVisitor may delete Inst, so keep track of whether it was a
1689 // terminator.
1690 bool IsTerminator = Inst->isTerminator();
1691 if (!DFSF.SkipInsts.count(V: Inst))
1692 DFSanVisitor(DFSF).visit(I: Inst);
1693 if (IsTerminator)
1694 break;
1695 Inst = Next;
1696 }
1697 }
1698
1699 // We will not necessarily be able to compute the shadow for every phi node
1700 // until we have visited every block. Therefore, the code that handles phi
1701 // nodes adds them to the PHIFixups list so that they can be properly
1702 // handled here.
1703 for (DFSanFunction::PHIFixupElement &P : DFSF.PHIFixups) {
1704 for (unsigned Val = 0, N = P.Phi->getNumIncomingValues(); Val != N;
1705 ++Val) {
1706 P.ShadowPhi->setIncomingValue(
1707 i: Val, V: DFSF.getShadow(V: P.Phi->getIncomingValue(i: Val)));
1708 if (P.OriginPhi)
1709 P.OriginPhi->setIncomingValue(
1710 i: Val, V: DFSF.getOrigin(V: P.Phi->getIncomingValue(i: Val)));
1711 }
1712 }
1713
1714 // -dfsan-debug-nonzero-labels will split the CFG in all kinds of crazy
1715 // places (i.e. instructions in basic blocks we haven't even begun visiting
1716 // yet). To make our life easier, do this work in a pass after the main
1717 // instrumentation.
1718 if (Opts.dfsan_debug_nonzero_labels) {
1719 for (Value *V : DFSF.NonZeroChecks) {
1720 BasicBlock::iterator Pos;
1721 if (Instruction *I = dyn_cast<Instruction>(Val: V))
1722 Pos = std::next(x: I->getIterator());
1723 else
1724 Pos = DFSF.F->getEntryBlock().begin();
1725 while (isa<PHINode>(Val: Pos) || isa<AllocaInst>(Val: Pos))
1726 Pos = std::next(x: Pos->getIterator());
1727 IRBuilder<> IRB(Pos);
1728 Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow: V, Pos);
1729 Value *Ne =
1730 IRB.CreateICmpNE(LHS: PrimitiveShadow, RHS: DFSF.DFS.ZeroPrimitiveShadow);
1731 UncondBrInst *BI = cast<UncondBrInst>(Val: SplitBlockAndInsertIfThen(
1732 Cond: Ne, SplitBefore: Pos, /*Unreachable=*/false, BranchWeights: ColdCallWeights));
1733 IRBuilder<> ThenIRB(BI);
1734 ThenIRB.CreateCall(Callee: DFSF.DFS.DFSanNonzeroLabelFn, Args: {});
1735 }
1736 }
1737 }
1738
1739 return Changed || !FnsToInstrument.empty() ||
1740 M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
1741}
1742
1743Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
1744 return IRB.CreatePtrAddWithUnsignedOffset(Ptr: DFS.ArgTLS, Offset: ArgOffset, Name: "_dfsarg");
1745}
1746
1747Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
1748 return IRB.CreatePointerCast(V: DFS.RetvalTLS, DestTy: PointerType::get(C&: *DFS.Ctx, AddressSpace: 0),
1749 Name: "_dfsret");
1750}
1751
1752Value *DFSanFunction::getRetvalOriginTLS() { return DFS.RetvalOriginTLS; }
1753
1754Value *DFSanFunction::getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB) {
1755 return IRB.CreateConstInBoundsGEP2_64(Ty: DFS.ArgOriginTLSTy, Ptr: DFS.ArgOriginTLS, Idx0: 0,
1756 Idx1: ArgNo, Name: "_dfsarg_o");
1757}
1758
1759Value *DFSanFunction::getOrigin(Value *V) {
1760 assert(DFS.shouldTrackOrigins());
1761 if (!isa<Argument>(Val: V) && !isa<Instruction>(Val: V))
1762 return DFS.ZeroOrigin;
1763 Value *&Origin = ValOriginMap[V];
1764 if (!Origin) {
1765 if (Argument *A = dyn_cast<Argument>(Val: V)) {
1766 if (IsNativeABI)
1767 return DFS.ZeroOrigin;
1768 if (A->getArgNo() < DFS.NumOfElementsInArgOrgTLS) {
1769 Instruction *ArgOriginTLSPos = &*F->getEntryBlock().begin();
1770 IRBuilder<> IRB(ArgOriginTLSPos);
1771 Value *ArgOriginPtr = getArgOriginTLS(ArgNo: A->getArgNo(), IRB);
1772 Origin = IRB.CreateLoad(Ty: DFS.OriginTy, Ptr: ArgOriginPtr);
1773 } else {
1774 // Overflow
1775 Origin = DFS.ZeroOrigin;
1776 }
1777 } else {
1778 Origin = DFS.ZeroOrigin;
1779 }
1780 }
1781 return Origin;
1782}
1783
1784void DFSanFunction::setOrigin(Instruction *I, Value *Origin) {
1785 if (!DFS.shouldTrackOrigins())
1786 return;
1787 assert(!ValOriginMap.count(I));
1788 assert(Origin->getType() == DFS.OriginTy);
1789 ValOriginMap[I] = Origin;
1790}
1791
1792Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
1793 unsigned ArgOffset = 0;
1794 const DataLayout &DL = F->getDataLayout();
1795 for (auto &FArg : F->args()) {
1796 if (!FArg.getType()->isSized()) {
1797 if (A == &FArg)
1798 break;
1799 continue;
1800 }
1801
1802 unsigned Size = DL.getTypeAllocSize(Ty: DFS.getShadowTy(V: &FArg));
1803 if (A != &FArg) {
1804 ArgOffset += alignTo(Size, A: ShadowTLSAlignment);
1805 if (ArgOffset > ArgTLSSize)
1806 break; // ArgTLS overflows, uses a zero shadow.
1807 continue;
1808 }
1809
1810 if (ArgOffset + Size > ArgTLSSize)
1811 break; // ArgTLS overflows, uses a zero shadow.
1812
1813 Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
1814 IRBuilder<> IRB(ArgTLSPos);
1815 Value *ArgShadowPtr = getArgTLS(T: FArg.getType(), ArgOffset, IRB);
1816 return IRB.CreateAlignedLoad(Ty: DFS.getShadowTy(V: &FArg), Ptr: ArgShadowPtr,
1817 Align: ShadowTLSAlignment);
1818 }
1819
1820 return DFS.getZeroShadow(V: A);
1821}
1822
1823Value *DFSanFunction::getShadow(Value *V) {
1824 if (!isa<Argument>(Val: V) && !isa<Instruction>(Val: V))
1825 return DFS.getZeroShadow(V);
1826 if (IsForceZeroLabels)
1827 return DFS.getZeroShadow(V);
1828 Value *&Shadow = ValShadowMap[V];
1829 if (!Shadow) {
1830 if (Argument *A = dyn_cast<Argument>(Val: V)) {
1831 if (IsNativeABI)
1832 return DFS.getZeroShadow(V);
1833 Shadow = getShadowForTLSArgument(A);
1834 NonZeroChecks.push_back(x: Shadow);
1835 } else {
1836 Shadow = DFS.getZeroShadow(V);
1837 }
1838 }
1839 return Shadow;
1840}
1841
1842void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
1843 assert(!ValShadowMap.count(I));
1844 ValShadowMap[I] = Shadow;
1845}
1846
1847/// Compute the integer shadow offset that corresponds to a given
1848/// application address.
1849///
1850/// Offset = (Addr & ~AndMask) ^ XorMask
1851Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) {
1852 assert(Addr != RetvalTLS && "Reinstrumenting?");
1853 Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1854
1855 uint64_t AndMask = MapParams->AndMask;
1856 if (AndMask)
1857 OffsetLong =
1858 IRB.CreateAnd(LHS: OffsetLong, RHS: ConstantInt::get(Ty: IntptrTy, V: ~AndMask));
1859
1860 uint64_t XorMask = MapParams->XorMask;
1861 if (XorMask)
1862 OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: ConstantInt::get(Ty: IntptrTy, V: XorMask));
1863 return OffsetLong;
1864}
1865
1866std::pair<Value *, Value *>
1867DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
1868 BasicBlock::iterator Pos) {
1869 // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL
1870 IRBuilder<> IRB(Pos);
1871 Value *ShadowOffset = getShadowOffset(Addr, IRB);
1872 Value *ShadowLong = ShadowOffset;
1873 uint64_t ShadowBase = MapParams->ShadowBase;
1874 if (ShadowBase != 0) {
1875 ShadowLong =
1876 IRB.CreateAdd(LHS: ShadowLong, RHS: ConstantInt::get(Ty: IntptrTy, V: ShadowBase));
1877 }
1878 Value *ShadowPtr = IRB.CreateIntToPtr(V: ShadowLong, DestTy: PointerType::get(C&: *Ctx, AddressSpace: 0));
1879 Value *OriginPtr = nullptr;
1880 if (shouldTrackOrigins()) {
1881 Value *OriginLong = ShadowOffset;
1882 uint64_t OriginBase = MapParams->OriginBase;
1883 if (OriginBase != 0)
1884 OriginLong =
1885 IRB.CreateAdd(LHS: OriginLong, RHS: ConstantInt::get(Ty: IntptrTy, V: OriginBase));
1886 const Align Alignment = llvm::assumeAligned(Value: InstAlignment.value());
1887 // When alignment is >= 4, Addr must be aligned to 4, otherwise it is UB.
1888 // So Mask is unnecessary.
1889 if (Alignment < MinOriginAlignment) {
1890 uint64_t Mask = MinOriginAlignment.value() - 1;
1891 OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: ConstantInt::get(Ty: IntptrTy, V: ~Mask));
1892 }
1893 OriginPtr = IRB.CreateIntToPtr(V: OriginLong, DestTy: OriginPtrTy);
1894 }
1895 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1896}
1897
1898Value *DataFlowSanitizer::getShadowAddress(Value *Addr,
1899 BasicBlock::iterator Pos,
1900 Value *ShadowOffset) {
1901 IRBuilder<> IRB(Pos);
1902 return IRB.CreateIntToPtr(V: ShadowOffset, DestTy: PrimitiveShadowPtrTy);
1903}
1904
1905Value *DataFlowSanitizer::getShadowAddress(Value *Addr,
1906 BasicBlock::iterator Pos) {
1907 IRBuilder<> IRB(Pos);
1908 Value *ShadowAddr = getShadowOffset(Addr, IRB);
1909 uint64_t ShadowBase = MapParams->ShadowBase;
1910 if (ShadowBase != 0)
1911 ShadowAddr =
1912 IRB.CreateAdd(LHS: ShadowAddr, RHS: ConstantInt::get(Ty: IntptrTy, V: ShadowBase));
1913 return getShadowAddress(Addr, Pos, ShadowOffset: ShadowAddr);
1914}
1915
1916Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
1917 BasicBlock::iterator Pos) {
1918 Value *PrimitiveValue = combineShadows(V1, V2, Pos);
1919 return expandFromPrimitiveShadow(T, PrimitiveShadow: PrimitiveValue, Pos);
1920}
1921
1922// Generates IR to compute the union of the two given shadows, inserting it
1923// before Pos. The combined value is with primitive type.
1924Value *DFSanFunction::combineShadows(Value *V1, Value *V2,
1925 BasicBlock::iterator Pos) {
1926 if (DFS.isZeroShadow(V: V1))
1927 return collapseToPrimitiveShadow(Shadow: V2, Pos);
1928 if (DFS.isZeroShadow(V: V2))
1929 return collapseToPrimitiveShadow(Shadow: V1, Pos);
1930 if (V1 == V2)
1931 return collapseToPrimitiveShadow(Shadow: V1, Pos);
1932
1933 auto V1Elems = ShadowElements.find(Val: V1);
1934 auto V2Elems = ShadowElements.find(Val: V2);
1935 if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
1936 if (llvm::includes(Range1&: V1Elems->second, Range2&: V2Elems->second)) {
1937 return collapseToPrimitiveShadow(Shadow: V1, Pos);
1938 }
1939 if (llvm::includes(Range1&: V2Elems->second, Range2&: V1Elems->second)) {
1940 return collapseToPrimitiveShadow(Shadow: V2, Pos);
1941 }
1942 } else if (V1Elems != ShadowElements.end()) {
1943 if (V1Elems->second.count(x: V2))
1944 return collapseToPrimitiveShadow(Shadow: V1, Pos);
1945 } else if (V2Elems != ShadowElements.end()) {
1946 if (V2Elems->second.count(x: V1))
1947 return collapseToPrimitiveShadow(Shadow: V2, Pos);
1948 }
1949
1950 auto Key = std::make_pair(x&: V1, y&: V2);
1951 if (V1 > V2)
1952 std::swap(a&: Key.first, b&: Key.second);
1953 CachedShadow &CCS = CachedShadows[Key];
1954 if (CCS.Block && DT.dominates(A: CCS.Block, B: Pos->getParent()))
1955 return CCS.Shadow;
1956
1957 // Converts inputs shadows to shadows with primitive types.
1958 Value *PV1 = collapseToPrimitiveShadow(Shadow: V1, Pos);
1959 Value *PV2 = collapseToPrimitiveShadow(Shadow: V2, Pos);
1960
1961 IRBuilder<> IRB(Pos);
1962 CCS.Block = Pos->getParent();
1963 CCS.Shadow = IRB.CreateOr(LHS: PV1, RHS: PV2);
1964
1965 std::set<Value *> UnionElems;
1966 if (V1Elems != ShadowElements.end()) {
1967 UnionElems = V1Elems->second;
1968 } else {
1969 UnionElems.insert(x: V1);
1970 }
1971 if (V2Elems != ShadowElements.end()) {
1972 UnionElems.insert(first: V2Elems->second.begin(), last: V2Elems->second.end());
1973 } else {
1974 UnionElems.insert(x: V2);
1975 }
1976 ShadowElements[CCS.Shadow] = std::move(UnionElems);
1977
1978 return CCS.Shadow;
1979}
1980
1981// A convenience function which folds the shadows of each of the operands
1982// of the provided instruction Inst, inserting the IR before Inst. Returns
1983// the computed union Value.
1984Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
1985 if (Inst->getNumOperands() == 0)
1986 return DFS.getZeroShadow(V: Inst);
1987
1988 Value *Shadow = getShadow(V: Inst->getOperand(i: 0));
1989 for (unsigned I = 1, N = Inst->getNumOperands(); I < N; ++I)
1990 Shadow = combineShadows(V1: Shadow, V2: getShadow(V: Inst->getOperand(i: I)),
1991 Pos: Inst->getIterator());
1992
1993 return expandFromPrimitiveShadow(T: Inst->getType(), PrimitiveShadow: Shadow,
1994 Pos: Inst->getIterator());
1995}
1996
1997void DFSanVisitor::visitInstOperands(Instruction &I) {
1998 Value *CombinedShadow = DFSF.combineOperandShadows(Inst: &I);
1999 DFSF.setShadow(I: &I, Shadow: CombinedShadow);
2000 visitInstOperandOrigins(I);
2001}
2002
2003Value *DFSanFunction::combineOrigins(const std::vector<Value *> &Shadows,
2004 const std::vector<Value *> &Origins,
2005 BasicBlock::iterator Pos,
2006 ConstantInt *Zero) {
2007 assert(Shadows.size() == Origins.size());
2008 size_t Size = Origins.size();
2009 if (Size == 0)
2010 return DFS.ZeroOrigin;
2011 Value *Origin = nullptr;
2012 if (!Zero)
2013 Zero = DFS.ZeroPrimitiveShadow;
2014 for (size_t I = 0; I != Size; ++I) {
2015 Value *OpOrigin = Origins[I];
2016 Constant *ConstOpOrigin = dyn_cast<Constant>(Val: OpOrigin);
2017 if (ConstOpOrigin && ConstOpOrigin->isNullValue())
2018 continue;
2019 if (!Origin) {
2020 Origin = OpOrigin;
2021 continue;
2022 }
2023 Value *OpShadow = Shadows[I];
2024 Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow: OpShadow, Pos);
2025 IRBuilder<> IRB(Pos);
2026 Value *Cond = IRB.CreateICmpNE(LHS: PrimitiveShadow, RHS: Zero);
2027 Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2028 }
2029 return Origin ? Origin : DFS.ZeroOrigin;
2030}
2031
2032Value *DFSanFunction::combineOperandOrigins(Instruction *Inst) {
2033 size_t Size = Inst->getNumOperands();
2034 std::vector<Value *> Shadows(Size);
2035 std::vector<Value *> Origins(Size);
2036 for (unsigned I = 0; I != Size; ++I) {
2037 Shadows[I] = getShadow(V: Inst->getOperand(i: I));
2038 Origins[I] = getOrigin(V: Inst->getOperand(i: I));
2039 }
2040 return combineOrigins(Shadows, Origins, Pos: Inst->getIterator());
2041}
2042
2043void DFSanVisitor::visitInstOperandOrigins(Instruction &I) {
2044 if (!DFSF.DFS.shouldTrackOrigins())
2045 return;
2046 Value *CombinedOrigin = DFSF.combineOperandOrigins(Inst: &I);
2047 DFSF.setOrigin(I: &I, Origin: CombinedOrigin);
2048}
2049
2050Align DFSanFunction::getShadowAlign(Align InstAlignment) {
2051 const Align Alignment =
2052 Opts.dfsan_preserve_alignment ? InstAlignment : Align(1);
2053 return Align(Alignment.value() * DFS.ShadowWidthBytes);
2054}
2055
2056Align DFSanFunction::getOriginAlign(Align InstAlignment) {
2057 const Align Alignment = llvm::assumeAligned(Value: InstAlignment.value());
2058 return Align(std::max(a: MinOriginAlignment, b: Alignment));
2059}
2060
2061bool DFSanFunction::isLookupTableConstant(Value *P) {
2062 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: P->stripPointerCasts()))
2063 if (GV->isConstant() && GV->hasName())
2064 return DFS.CombineTaintLookupTableNames.count(Key: GV->getName());
2065
2066 return false;
2067}
2068
2069bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size,
2070 Align InstAlignment) {
2071 // When enabling tracking load instructions, we always use
2072 // __dfsan_load_label_and_origin to reduce code size.
2073 if (Opts.dfsan_track_origins == 2)
2074 return true;
2075
2076 assert(Size != 0);
2077 // * if Size == 1, it is sufficient to load its origin aligned at 4.
2078 // * if Size == 2, we assume most cases Addr % 2 == 0, so it is sufficient to
2079 // load its origin aligned at 4. If not, although origins may be lost, it
2080 // should not happen very often.
2081 // * if align >= 4, Addr must be aligned to 4, otherwise it is UB. When
2082 // Size % 4 == 0, it is more efficient to load origins without callbacks.
2083 // * Otherwise we use __dfsan_load_label_and_origin.
2084 // This should ensure that common cases run efficiently.
2085 if (Size <= 2)
2086 return false;
2087
2088 const Align Alignment = llvm::assumeAligned(Value: InstAlignment.value());
2089 return Alignment < MinOriginAlignment || !DFS.hasLoadSizeForFastPath(Size);
2090}
2091
2092Value *DataFlowSanitizer::loadNextOrigin(BasicBlock::iterator Pos,
2093 Align OriginAlign,
2094 Value **OriginAddr) {
2095 IRBuilder<> IRB(Pos);
2096 IRB.disableGEPCanonicalization();
2097 *OriginAddr =
2098 IRB.CreateGEP(Ty: OriginTy, Ptr: *OriginAddr, IdxList: ConstantInt::get(Ty: IntptrTy, V: 1));
2099 return IRB.CreateAlignedLoad(Ty: OriginTy, Ptr: *OriginAddr, Align: OriginAlign);
2100}
2101
2102std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
2103 Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
2104 Align OriginAlign, Value *FirstOrigin, BasicBlock::iterator Pos) {
2105 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2106 const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
2107
2108 assert(Size >= 4 && "Not large enough load size for fast path!");
2109
2110 // Used for origin tracking.
2111 std::vector<Value *> Shadows;
2112 std::vector<Value *> Origins;
2113
2114 // Load instructions in LLVM can have arbitrary byte sizes (e.g., 3, 12, 20)
2115 // but this function is only used in a subset of cases that make it possible
2116 // to optimize the instrumentation.
2117 //
2118 // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
2119 // per byte) is either:
2120 // - a multiple of 8 (common)
2121 // - equal to 4 (only for load32)
2122 //
2123 // For the second case, we can fit the wide shadow in a 32-bit integer. In all
2124 // other cases, we use a 64-bit integer to hold the wide shadow.
2125 Type *WideShadowTy =
2126 ShadowSize == 4 ? Type::getInt32Ty(C&: *DFS.Ctx) : Type::getInt64Ty(C&: *DFS.Ctx);
2127
2128 IRBuilder<> IRB(Pos);
2129 IRB.disableGEPCanonicalization();
2130 Value *CombinedWideShadow =
2131 IRB.CreateAlignedLoad(Ty: WideShadowTy, Ptr: ShadowAddr, Align: ShadowAlign);
2132
2133 unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
2134 const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
2135
2136 auto AppendWideShadowAndOrigin = [&](Value *WideShadow, Value *Origin) {
2137 if (BytesPerWideShadow > 4) {
2138 assert(BytesPerWideShadow == 8);
2139 // The wide shadow relates to two origin pointers: one for the first four
2140 // application bytes, and one for the latest four. We use a left shift to
2141 // get just the shadow bytes that correspond to the first origin pointer,
2142 // and then the entire shadow for the second origin pointer (which will be
2143 // chosen by combineOrigins() iff the least-significant half of the wide
2144 // shadow was empty but the other half was not).
2145 Value *WideShadowLo =
2146 F->getDataLayout().isLittleEndian()
2147 ? IRB.CreateShl(
2148 LHS: WideShadow,
2149 RHS: ConstantInt::get(Ty: WideShadowTy, V: WideShadowBitWidth / 2))
2150 : IRB.CreateAnd(
2151 LHS: WideShadow,
2152 RHS: ConstantInt::get(Ty: WideShadowTy,
2153 V: ((1ULL << (WideShadowBitWidth / 2)) - 1)
2154 << (WideShadowBitWidth / 2)));
2155 Shadows.push_back(x: WideShadow);
2156 Origins.push_back(x: DFS.loadNextOrigin(Pos, OriginAlign, OriginAddr: &OriginAddr));
2157
2158 Shadows.push_back(x: WideShadowLo);
2159 Origins.push_back(x: Origin);
2160 } else {
2161 Shadows.push_back(x: WideShadow);
2162 Origins.push_back(x: Origin);
2163 }
2164 };
2165
2166 if (ShouldTrackOrigins)
2167 AppendWideShadowAndOrigin(CombinedWideShadow, FirstOrigin);
2168
2169 // First OR all the WideShadows (i.e., 64bit or 32bit shadow chunks) linearly;
2170 // then OR individual shadows within the combined WideShadow by binary ORing.
2171 // This is fewer instructions than ORing shadows individually, since it
2172 // needs logN shift/or instructions (N being the bytes of the combined wide
2173 // shadow).
2174 for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
2175 ByteOfs += BytesPerWideShadow) {
2176 ShadowAddr = IRB.CreateGEP(Ty: WideShadowTy, Ptr: ShadowAddr,
2177 IdxList: ConstantInt::get(Ty: DFS.IntptrTy, V: 1));
2178 Value *NextWideShadow =
2179 IRB.CreateAlignedLoad(Ty: WideShadowTy, Ptr: ShadowAddr, Align: ShadowAlign);
2180 CombinedWideShadow = IRB.CreateOr(LHS: CombinedWideShadow, RHS: NextWideShadow);
2181 if (ShouldTrackOrigins) {
2182 Value *NextOrigin = DFS.loadNextOrigin(Pos, OriginAlign, OriginAddr: &OriginAddr);
2183 AppendWideShadowAndOrigin(NextWideShadow, NextOrigin);
2184 }
2185 }
2186 for (unsigned Width = WideShadowBitWidth / 2; Width >= DFS.ShadowWidthBits;
2187 Width >>= 1) {
2188 Value *ShrShadow = IRB.CreateLShr(LHS: CombinedWideShadow, RHS: Width);
2189 CombinedWideShadow = IRB.CreateOr(LHS: CombinedWideShadow, RHS: ShrShadow);
2190 }
2191 return {IRB.CreateTrunc(V: CombinedWideShadow, DestTy: DFS.PrimitiveShadowTy),
2192 ShouldTrackOrigins
2193 ? combineOrigins(Shadows, Origins, Pos,
2194 Zero: ConstantInt::getSigned(Ty: IRB.getInt64Ty(), V: 0))
2195 : DFS.ZeroOrigin};
2196}
2197
2198std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
2199 Value *Addr, uint64_t Size, Align InstAlignment, BasicBlock::iterator Pos) {
2200 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2201
2202 // Non-escaped loads.
2203 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: Addr)) {
2204 const auto SI = AllocaShadowMap.find(Val: AI);
2205 if (SI != AllocaShadowMap.end()) {
2206 IRBuilder<> IRB(Pos);
2207 Value *ShadowLI = IRB.CreateLoad(Ty: DFS.PrimitiveShadowTy, Ptr: SI->second);
2208 const auto OI = AllocaOriginMap.find(Val: AI);
2209 assert(!ShouldTrackOrigins || OI != AllocaOriginMap.end());
2210 return {ShadowLI, ShouldTrackOrigins
2211 ? IRB.CreateLoad(Ty: DFS.OriginTy, Ptr: OI->second)
2212 : nullptr};
2213 }
2214 }
2215
2216 // Load from constant addresses.
2217 SmallVector<const Value *, 2> Objs;
2218 getUnderlyingObjects(V: Addr, Objects&: Objs);
2219 bool AllConstants = true;
2220 for (const Value *Obj : Objs) {
2221 if (isa<Function>(Val: Obj) || isa<BlockAddress>(Val: Obj))
2222 continue;
2223 if (isa<GlobalVariable>(Val: Obj) && cast<GlobalVariable>(Val: Obj)->isConstant())
2224 continue;
2225
2226 AllConstants = false;
2227 break;
2228 }
2229 if (AllConstants)
2230 return {DFS.ZeroPrimitiveShadow,
2231 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2232
2233 if (Size == 0)
2234 return {DFS.ZeroPrimitiveShadow,
2235 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2236
2237 // Use callback to load if this is not an optimizable case for origin
2238 // tracking.
2239 if (ShouldTrackOrigins &&
2240 useCallbackLoadLabelAndOrigin(Size, InstAlignment)) {
2241 IRBuilder<> IRB(Pos);
2242 CallInst *Call =
2243 IRB.CreateCall(Callee: DFS.DFSanLoadLabelAndOriginFn,
2244 Args: {Addr, ConstantInt::get(Ty: DFS.IntptrTy, V: Size)});
2245 Call->addRetAttr(Kind: Attribute::ZExt);
2246 return {IRB.CreateTrunc(V: IRB.CreateLShr(LHS: Call, RHS: DFS.OriginWidthBits),
2247 DestTy: DFS.PrimitiveShadowTy),
2248 IRB.CreateTrunc(V: Call, DestTy: DFS.OriginTy)};
2249 }
2250
2251 // Other cases that support loading shadows or origins in a fast way.
2252 Value *ShadowAddr, *OriginAddr;
2253 std::tie(args&: ShadowAddr, args&: OriginAddr) =
2254 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2255
2256 const Align ShadowAlign = getShadowAlign(InstAlignment);
2257 const Align OriginAlign = getOriginAlign(InstAlignment);
2258 Value *Origin = nullptr;
2259 if (ShouldTrackOrigins) {
2260 IRBuilder<> IRB(Pos);
2261 Origin = IRB.CreateAlignedLoad(Ty: DFS.OriginTy, Ptr: OriginAddr, Align: OriginAlign);
2262 }
2263
2264 // When the byte size is small enough, we can load the shadow directly with
2265 // just a few instructions.
2266 switch (Size) {
2267 case 1: {
2268 LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
2269 LI->setAlignment(ShadowAlign);
2270 return {LI, Origin};
2271 }
2272 case 2: {
2273 IRBuilder<> IRB(Pos);
2274 IRB.disableGEPCanonicalization();
2275 Value *ShadowAddr1 = IRB.CreateGEP(Ty: DFS.PrimitiveShadowTy, Ptr: ShadowAddr,
2276 IdxList: ConstantInt::get(Ty: DFS.IntptrTy, V: 1));
2277 Value *Load =
2278 IRB.CreateAlignedLoad(Ty: DFS.PrimitiveShadowTy, Ptr: ShadowAddr, Align: ShadowAlign);
2279 Value *Load1 =
2280 IRB.CreateAlignedLoad(Ty: DFS.PrimitiveShadowTy, Ptr: ShadowAddr1, Align: ShadowAlign);
2281 return {combineShadows(V1: Load, V2: Load1, Pos), Origin};
2282 }
2283 }
2284 bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(Size);
2285
2286 if (HasSizeForFastPath)
2287 return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
2288 OriginAlign, FirstOrigin: Origin, Pos);
2289
2290 IRBuilder<> IRB(Pos);
2291 CallInst *FallbackCall = IRB.CreateCall(
2292 Callee: DFS.DFSanUnionLoadFn, Args: {ShadowAddr, ConstantInt::get(Ty: DFS.IntptrTy, V: Size)});
2293 FallbackCall->addRetAttr(Kind: Attribute::ZExt);
2294 return {FallbackCall, Origin};
2295}
2296
2297std::pair<Value *, Value *>
2298DFSanFunction::loadShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment,
2299 BasicBlock::iterator Pos) {
2300 Value *PrimitiveShadow, *Origin;
2301 std::tie(args&: PrimitiveShadow, args&: Origin) =
2302 loadShadowOriginSansLoadTracking(Addr, Size, InstAlignment, Pos);
2303 if (DFS.shouldTrackOrigins()) {
2304 if (Opts.dfsan_track_origins == 2) {
2305 IRBuilder<> IRB(Pos);
2306 auto *ConstantShadow = dyn_cast<Constant>(Val: PrimitiveShadow);
2307 if (!ConstantShadow || !ConstantShadow->isNullValue())
2308 Origin = updateOriginIfTainted(Shadow: PrimitiveShadow, Origin, IRB);
2309 }
2310 }
2311 return {PrimitiveShadow, Origin};
2312}
2313
2314static AtomicOrdering addAcquireOrdering(AtomicOrdering AO) {
2315 switch (AO) {
2316 case AtomicOrdering::NotAtomic:
2317 return AtomicOrdering::NotAtomic;
2318 case AtomicOrdering::Unordered:
2319 case AtomicOrdering::Monotonic:
2320 case AtomicOrdering::Acquire:
2321 return AtomicOrdering::Acquire;
2322 case AtomicOrdering::Release:
2323 case AtomicOrdering::AcquireRelease:
2324 return AtomicOrdering::AcquireRelease;
2325 case AtomicOrdering::SequentiallyConsistent:
2326 return AtomicOrdering::SequentiallyConsistent;
2327 }
2328 llvm_unreachable("Unknown ordering");
2329}
2330
2331Value *StripPointerGEPsAndCasts(Value *V) {
2332 if (!V->getType()->isPointerTy())
2333 return V;
2334
2335 // DFSan pass should be running on valid IR, but we'll
2336 // keep a seen set to ensure there are no issues.
2337 SmallPtrSet<const Value *, 4> Visited;
2338 Visited.insert(Ptr: V);
2339 do {
2340 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
2341 V = GEP->getPointerOperand();
2342 } else if (Operator::getOpcode(V) == Instruction::BitCast) {
2343 V = cast<Operator>(Val: V)->getOperand(i: 0);
2344 if (!V->getType()->isPointerTy())
2345 return V;
2346 } else if (isa<GlobalAlias>(Val: V)) {
2347 V = cast<GlobalAlias>(Val: V)->getAliasee();
2348 }
2349 } while (Visited.insert(Ptr: V).second);
2350
2351 return V;
2352}
2353
2354void DFSanVisitor::visitLoadInst(LoadInst &LI) {
2355 auto &DL = LI.getDataLayout();
2356 uint64_t Size = DL.getTypeStoreSize(Ty: LI.getType());
2357 if (Size == 0) {
2358 DFSF.setShadow(I: &LI, Shadow: DFSF.DFS.getZeroShadow(V: &LI));
2359 DFSF.setOrigin(I: &LI, Origin: DFSF.DFS.ZeroOrigin);
2360 return;
2361 }
2362
2363 // When an application load is atomic, increase atomic ordering between
2364 // atomic application loads and stores to ensure happen-before order; load
2365 // shadow data after application data; store zero shadow data before
2366 // application data. This ensure shadow loads return either labels of the
2367 // initial application data or zeros.
2368 if (LI.isAtomic())
2369 LI.setOrdering(addAcquireOrdering(AO: LI.getOrdering()));
2370
2371 BasicBlock::iterator AfterLi = std::next(x: LI.getIterator());
2372 BasicBlock::iterator Pos = LI.getIterator();
2373 if (LI.isAtomic())
2374 Pos = std::next(x: Pos);
2375
2376 std::vector<Value *> Shadows;
2377 std::vector<Value *> Origins;
2378 Value *PrimitiveShadow, *Origin;
2379 std::tie(args&: PrimitiveShadow, args&: Origin) =
2380 DFSF.loadShadowOrigin(Addr: LI.getPointerOperand(), Size, InstAlignment: LI.getAlign(), Pos);
2381 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2382 if (ShouldTrackOrigins) {
2383 Shadows.push_back(x: PrimitiveShadow);
2384 Origins.push_back(x: Origin);
2385 }
2386 if (DFSF.Opts.dfsan_combine_pointer_labels_on_load ||
2387 DFSF.isLookupTableConstant(
2388 P: StripPointerGEPsAndCasts(V: LI.getPointerOperand()))) {
2389 Value *PtrShadow = DFSF.getShadow(V: LI.getPointerOperand());
2390 PrimitiveShadow = DFSF.combineShadows(V1: PrimitiveShadow, V2: PtrShadow, Pos);
2391 if (ShouldTrackOrigins) {
2392 Shadows.push_back(x: PtrShadow);
2393 Origins.push_back(x: DFSF.getOrigin(V: LI.getPointerOperand()));
2394 }
2395 }
2396 if (!DFSF.DFS.isZeroShadow(V: PrimitiveShadow))
2397 DFSF.NonZeroChecks.push_back(x: PrimitiveShadow);
2398
2399 Value *Shadow =
2400 DFSF.expandFromPrimitiveShadow(T: LI.getType(), PrimitiveShadow, Pos);
2401 DFSF.setShadow(I: &LI, Shadow);
2402
2403 if (ShouldTrackOrigins) {
2404 DFSF.setOrigin(I: &LI, Origin: DFSF.combineOrigins(Shadows, Origins, Pos));
2405 }
2406
2407 if (DFSF.Opts.dfsan_event_callbacks) {
2408 IRBuilder<> IRB(Pos);
2409 Value *Addr = LI.getPointerOperand();
2410 CallInst *CI =
2411 IRB.CreateCall(Callee: DFSF.DFS.DFSanLoadCallbackFn, Args: {PrimitiveShadow, Addr});
2412 CI->maybeAddParamAttr(ArgNo: 0, Kind: DFSF.TLI.getExtAttrForI8Param(/*Signed=*/false));
2413 }
2414
2415 IRBuilder<> IRB(AfterLi);
2416 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, I&: LI, Data: &LI);
2417}
2418
2419Value *DFSanFunction::updateOriginIfTainted(Value *Shadow, Value *Origin,
2420 IRBuilder<> &IRB) {
2421 assert(DFS.shouldTrackOrigins());
2422 return IRB.CreateCall(Callee: DFS.DFSanChainOriginIfTaintedFn, Args: {Shadow, Origin});
2423}
2424
2425Value *DFSanFunction::updateOrigin(Value *V, IRBuilder<> &IRB) {
2426 if (!DFS.shouldTrackOrigins())
2427 return V;
2428 return IRB.CreateCall(Callee: DFS.DFSanChainOriginFn, Args: V);
2429}
2430
2431Value *DFSanFunction::originToIntptr(IRBuilder<> &IRB, Value *Origin) {
2432 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2433 const DataLayout &DL = F->getDataLayout();
2434 unsigned IntptrSize = DL.getTypeStoreSize(Ty: DFS.IntptrTy);
2435 if (IntptrSize == OriginSize)
2436 return Origin;
2437 assert(IntptrSize == OriginSize * 2);
2438 Origin = IRB.CreateIntCast(V: Origin, DestTy: DFS.IntptrTy, /* isSigned */ false);
2439 return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: OriginSize * 8));
2440}
2441
2442void DFSanFunction::paintOrigin(IRBuilder<> &IRB, Value *Origin,
2443 Value *StoreOriginAddr,
2444 uint64_t StoreOriginSize, Align Alignment) {
2445 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2446 const DataLayout &DL = F->getDataLayout();
2447 const Align IntptrAlignment = DL.getABITypeAlign(Ty: DFS.IntptrTy);
2448 unsigned IntptrSize = DL.getTypeStoreSize(Ty: DFS.IntptrTy);
2449 assert(IntptrAlignment >= MinOriginAlignment);
2450 assert(IntptrSize >= OriginSize);
2451
2452 unsigned Ofs = 0;
2453 Align CurrentAlignment = Alignment;
2454 if (Alignment >= IntptrAlignment && IntptrSize > OriginSize) {
2455 Value *IntptrOrigin = originToIntptr(IRB, Origin);
2456 Value *IntptrStoreOriginPtr =
2457 IRB.CreatePointerCast(V: StoreOriginAddr, DestTy: PointerType::get(C&: *DFS.Ctx, AddressSpace: 0));
2458 for (unsigned I = 0; I < StoreOriginSize / IntptrSize; ++I) {
2459 Value *Ptr =
2460 I ? IRB.CreateConstGEP1_32(Ty: DFS.IntptrTy, Ptr: IntptrStoreOriginPtr, Idx0: I)
2461 : IntptrStoreOriginPtr;
2462 IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
2463 Ofs += IntptrSize / OriginSize;
2464 CurrentAlignment = IntptrAlignment;
2465 }
2466 }
2467
2468 for (unsigned I = Ofs; I < (StoreOriginSize + OriginSize - 1) / OriginSize;
2469 ++I) {
2470 Value *GEP = I ? IRB.CreateConstGEP1_32(Ty: DFS.OriginTy, Ptr: StoreOriginAddr, Idx0: I)
2471 : StoreOriginAddr;
2472 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
2473 CurrentAlignment = MinOriginAlignment;
2474 }
2475}
2476
2477Value *DFSanFunction::convertToBool(Value *V, IRBuilder<> &IRB,
2478 const Twine &Name) {
2479 Type *VTy = V->getType();
2480 assert(VTy->isIntegerTy());
2481 if (VTy->getIntegerBitWidth() == 1)
2482 // Just converting a bool to a bool, so do nothing.
2483 return V;
2484 return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name);
2485}
2486
2487void DFSanFunction::storeOrigin(BasicBlock::iterator Pos, Value *Addr,
2488 uint64_t Size, Value *Shadow, Value *Origin,
2489 Value *StoreOriginAddr, Align InstAlignment) {
2490 // Do not write origins for zero shadows because we do not trace origins for
2491 // untainted sinks.
2492 const Align OriginAlignment = getOriginAlign(InstAlignment);
2493 Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos);
2494 IRBuilder<> IRB(Pos);
2495 IRB.disableGEPCanonicalization();
2496 if (auto *ConstantShadow = dyn_cast<Constant>(Val: CollapsedShadow)) {
2497 if (!ConstantShadow->isNullValue())
2498 paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), StoreOriginAddr, StoreOriginSize: Size,
2499 Alignment: OriginAlignment);
2500 return;
2501 }
2502
2503 if (shouldInstrumentWithCall()) {
2504 IRB.CreateCall(
2505 Callee: DFS.DFSanMaybeStoreOriginFn,
2506 Args: {CollapsedShadow, Addr, ConstantInt::get(Ty: DFS.IntptrTy, V: Size), Origin});
2507 } else {
2508 Value *Cmp = convertToBool(V: CollapsedShadow, IRB, Name: "_dfscmp");
2509 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
2510 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
2511 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: DFS.OriginStoreWeights, DTU: &DTU);
2512 IRBuilder<> IRBNew(CheckTerm);
2513 IRBNew.disableGEPCanonicalization();
2514 paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), StoreOriginAddr, StoreOriginSize: Size,
2515 Alignment: OriginAlignment);
2516 ++NumOriginStores;
2517 }
2518}
2519
2520void DFSanFunction::storeZeroPrimitiveShadow(Value *Addr, uint64_t Size,
2521 Align ShadowAlign,
2522 BasicBlock::iterator Pos) {
2523 IRBuilder<> IRB(Pos);
2524 IntegerType *ShadowTy =
2525 IntegerType::get(C&: *DFS.Ctx, NumBits: Size * DFS.ShadowWidthBits);
2526 Value *ExtZeroShadow = ConstantInt::get(Ty: ShadowTy, V: 0);
2527 Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
2528 IRB.CreateAlignedStore(Val: ExtZeroShadow, Ptr: ShadowAddr, Align: ShadowAlign);
2529 // Do not write origins for 0 shadows because we do not trace origins for
2530 // untainted sinks.
2531}
2532
2533void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
2534 Align InstAlignment,
2535 Value *PrimitiveShadow,
2536 Value *Origin,
2537 BasicBlock::iterator Pos) {
2538 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin;
2539
2540 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: Addr)) {
2541 const auto SI = AllocaShadowMap.find(Val: AI);
2542 if (SI != AllocaShadowMap.end()) {
2543 IRBuilder<> IRB(Pos);
2544 IRB.CreateStore(Val: PrimitiveShadow, Ptr: SI->second);
2545
2546 // Do not write origins for 0 shadows because we do not trace origins for
2547 // untainted sinks.
2548 if (ShouldTrackOrigins && !DFS.isZeroShadow(V: PrimitiveShadow)) {
2549 const auto OI = AllocaOriginMap.find(Val: AI);
2550 assert(OI != AllocaOriginMap.end() && Origin);
2551 IRB.CreateStore(Val: Origin, Ptr: OI->second);
2552 }
2553 return;
2554 }
2555 }
2556
2557 const Align ShadowAlign = getShadowAlign(InstAlignment);
2558 if (DFS.isZeroShadow(V: PrimitiveShadow)) {
2559 storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, Pos);
2560 return;
2561 }
2562
2563 IRBuilder<> IRB(Pos);
2564 IRB.disableGEPCanonicalization();
2565 Value *ShadowAddr, *OriginAddr;
2566 std::tie(args&: ShadowAddr, args&: OriginAddr) =
2567 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2568
2569 const unsigned ShadowVecSize = 8;
2570 assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
2571 "Shadow vector is too large!");
2572
2573 uint64_t Offset = 0;
2574 uint64_t LeftSize = Size;
2575 if (LeftSize >= ShadowVecSize) {
2576 auto *ShadowVecTy =
2577 FixedVectorType::get(ElementType: DFS.PrimitiveShadowTy, NumElts: ShadowVecSize);
2578 Value *ShadowVec = PoisonValue::get(T: ShadowVecTy);
2579 for (unsigned I = 0; I != ShadowVecSize; ++I) {
2580 ShadowVec = IRB.CreateInsertElement(
2581 Vec: ShadowVec, NewElt: PrimitiveShadow,
2582 Idx: ConstantInt::get(Ty: Type::getInt32Ty(C&: *DFS.Ctx), V: I));
2583 }
2584 do {
2585 Value *CurShadowVecAddr =
2586 IRB.CreateConstGEP1_32(Ty: ShadowVecTy, Ptr: ShadowAddr, Idx0: Offset);
2587 IRB.CreateAlignedStore(Val: ShadowVec, Ptr: CurShadowVecAddr, Align: ShadowAlign);
2588 LeftSize -= ShadowVecSize;
2589 ++Offset;
2590 } while (LeftSize >= ShadowVecSize);
2591 Offset *= ShadowVecSize;
2592 }
2593 while (LeftSize > 0) {
2594 Value *CurShadowAddr =
2595 IRB.CreateConstGEP1_32(Ty: DFS.PrimitiveShadowTy, Ptr: ShadowAddr, Idx0: Offset);
2596 IRB.CreateAlignedStore(Val: PrimitiveShadow, Ptr: CurShadowAddr, Align: ShadowAlign);
2597 --LeftSize;
2598 ++Offset;
2599 }
2600
2601 if (ShouldTrackOrigins) {
2602 storeOrigin(Pos, Addr, Size, Shadow: PrimitiveShadow, Origin, StoreOriginAddr: OriginAddr,
2603 InstAlignment);
2604 }
2605}
2606
2607static AtomicOrdering addReleaseOrdering(AtomicOrdering AO) {
2608 switch (AO) {
2609 case AtomicOrdering::NotAtomic:
2610 return AtomicOrdering::NotAtomic;
2611 case AtomicOrdering::Unordered:
2612 case AtomicOrdering::Monotonic:
2613 case AtomicOrdering::Release:
2614 return AtomicOrdering::Release;
2615 case AtomicOrdering::Acquire:
2616 case AtomicOrdering::AcquireRelease:
2617 return AtomicOrdering::AcquireRelease;
2618 case AtomicOrdering::SequentiallyConsistent:
2619 return AtomicOrdering::SequentiallyConsistent;
2620 }
2621 llvm_unreachable("Unknown ordering");
2622}
2623
2624void DFSanVisitor::visitStoreInst(StoreInst &SI) {
2625 auto &DL = SI.getDataLayout();
2626 Value *Val = SI.getValueOperand();
2627 uint64_t Size = DL.getTypeStoreSize(Ty: Val->getType());
2628 if (Size == 0)
2629 return;
2630
2631 // When an application store is atomic, increase atomic ordering between
2632 // atomic application loads and stores to ensure happen-before order; load
2633 // shadow data after application data; store zero shadow data before
2634 // application data. This ensure shadow loads return either labels of the
2635 // initial application data or zeros.
2636 if (SI.isAtomic())
2637 SI.setOrdering(addReleaseOrdering(AO: SI.getOrdering()));
2638
2639 const bool ShouldTrackOrigins =
2640 DFSF.DFS.shouldTrackOrigins() && !SI.isAtomic();
2641 std::vector<Value *> Shadows;
2642 std::vector<Value *> Origins;
2643
2644 Value *Shadow =
2645 SI.isAtomic() ? DFSF.DFS.getZeroShadow(V: Val) : DFSF.getShadow(V: Val);
2646
2647 if (ShouldTrackOrigins) {
2648 Shadows.push_back(x: Shadow);
2649 Origins.push_back(x: DFSF.getOrigin(V: Val));
2650 }
2651
2652 Value *PrimitiveShadow;
2653 if (DFSF.Opts.dfsan_combine_pointer_labels_on_store) {
2654 Value *PtrShadow = DFSF.getShadow(V: SI.getPointerOperand());
2655 if (ShouldTrackOrigins) {
2656 Shadows.push_back(x: PtrShadow);
2657 Origins.push_back(x: DFSF.getOrigin(V: SI.getPointerOperand()));
2658 }
2659 PrimitiveShadow = DFSF.combineShadows(V1: Shadow, V2: PtrShadow, Pos: SI.getIterator());
2660 } else {
2661 PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, Pos: SI.getIterator());
2662 }
2663 Value *Origin = nullptr;
2664 if (ShouldTrackOrigins)
2665 Origin = DFSF.combineOrigins(Shadows, Origins, Pos: SI.getIterator());
2666 DFSF.storePrimitiveShadowOrigin(Addr: SI.getPointerOperand(), Size, InstAlignment: SI.getAlign(),
2667 PrimitiveShadow, Origin, Pos: SI.getIterator());
2668 if (DFSF.Opts.dfsan_event_callbacks) {
2669 IRBuilder<> IRB(&SI);
2670 Value *Addr = SI.getPointerOperand();
2671 CallInst *CI =
2672 IRB.CreateCall(Callee: DFSF.DFS.DFSanStoreCallbackFn, Args: {PrimitiveShadow, Addr});
2673 CI->maybeAddParamAttr(ArgNo: 0, Kind: DFSF.TLI.getExtAttrForI8Param(/*Signed=*/false));
2674 }
2675}
2676
2677void DFSanVisitor::visitCASOrRMW(Align InstAlignment, Instruction &I) {
2678 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2679
2680 Value *Val = I.getOperand(i: 1);
2681 const auto &DL = I.getDataLayout();
2682 uint64_t Size = DL.getTypeStoreSize(Ty: Val->getType());
2683 if (Size == 0)
2684 return;
2685
2686 // Conservatively set data at stored addresses and return with zero shadow to
2687 // prevent shadow data races.
2688 Value *Addr = I.getOperand(i: 0);
2689 const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment);
2690 DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, Pos: I.getIterator());
2691 DFSF.setShadow(I: &I, Shadow: DFSF.DFS.getZeroShadow(V: &I));
2692 DFSF.setOrigin(I: &I, Origin: DFSF.DFS.ZeroOrigin);
2693}
2694
2695void DFSanVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
2696 visitCASOrRMW(InstAlignment: I.getAlign(), I);
2697 // TODO: The ordering change follows MSan. It is possible not to change
2698 // ordering because we always set and use 0 shadows.
2699 I.setOrdering(addReleaseOrdering(AO: I.getOrdering()));
2700}
2701
2702void DFSanVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2703 visitCASOrRMW(InstAlignment: I.getAlign(), I);
2704 // TODO: The ordering change follows MSan. It is possible not to change
2705 // ordering because we always set and use 0 shadows.
2706 I.setSuccessOrdering(addReleaseOrdering(AO: I.getSuccessOrdering()));
2707}
2708
2709void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) {
2710 visitInstOperands(I&: UO);
2711}
2712
2713void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) {
2714 visitInstOperands(I&: BO);
2715}
2716
2717void DFSanVisitor::visitBitCastInst(BitCastInst &BCI) {
2718 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2719 // a musttail call and a ret, don't instrument. New instructions are not
2720 // allowed after a musttail call.
2721 if (auto *CI = dyn_cast<CallInst>(Val: BCI.getOperand(i_nocapture: 0)))
2722 if (CI->isMustTailCall())
2723 return;
2724 visitInstOperands(I&: BCI);
2725}
2726
2727void DFSanVisitor::visitCastInst(CastInst &CI) { visitInstOperands(I&: CI); }
2728
2729void DFSanVisitor::visitCmpInst(CmpInst &CI) {
2730 visitInstOperands(I&: CI);
2731 if (DFSF.Opts.dfsan_event_callbacks) {
2732 IRBuilder<> IRB(&CI);
2733 Value *CombinedShadow = DFSF.getShadow(V: &CI);
2734 CallInst *CallI =
2735 IRB.CreateCall(Callee: DFSF.DFS.DFSanCmpCallbackFn, Args: CombinedShadow);
2736 CallI->maybeAddParamAttr(ArgNo: 0,
2737 Kind: DFSF.TLI.getExtAttrForI8Param(/*Signed=*/false));
2738 }
2739}
2740
2741void DFSanVisitor::visitLandingPadInst(LandingPadInst &LPI) {
2742 // We do not need to track data through LandingPadInst.
2743 //
2744 // For the C++ exceptions, if a value is thrown, this value will be stored
2745 // in a memory location provided by __cxa_allocate_exception(...) (on the
2746 // throw side) or __cxa_begin_catch(...) (on the catch side).
2747 // This memory will have a shadow, so with the loads and stores we will be
2748 // able to propagate labels on data thrown through exceptions, without any
2749 // special handling of the LandingPadInst.
2750 //
2751 // The second element in the pair result of the LandingPadInst is a
2752 // register value, but it is for a type ID and should never be tainted.
2753 DFSF.setShadow(I: &LPI, Shadow: DFSF.DFS.getZeroShadow(V: &LPI));
2754 DFSF.setOrigin(I: &LPI, Origin: DFSF.DFS.ZeroOrigin);
2755}
2756
2757void DFSanVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
2758 if (DFSF.Opts.dfsan_combine_offset_labels_on_gep ||
2759 DFSF.isLookupTableConstant(
2760 P: StripPointerGEPsAndCasts(V: GEPI.getPointerOperand()))) {
2761 visitInstOperands(I&: GEPI);
2762 return;
2763 }
2764
2765 // Only propagate shadow/origin of base pointer value but ignore those of
2766 // offset operands.
2767 Value *BasePointer = GEPI.getPointerOperand();
2768 DFSF.setShadow(I: &GEPI, Shadow: DFSF.getShadow(V: BasePointer));
2769 if (DFSF.DFS.shouldTrackOrigins())
2770 DFSF.setOrigin(I: &GEPI, Origin: DFSF.getOrigin(V: BasePointer));
2771}
2772
2773void DFSanVisitor::visitExtractElementInst(ExtractElementInst &I) {
2774 visitInstOperands(I);
2775}
2776
2777void DFSanVisitor::visitInsertElementInst(InsertElementInst &I) {
2778 visitInstOperands(I);
2779}
2780
2781void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
2782 visitInstOperands(I);
2783}
2784
2785void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
2786 IRBuilder<> IRB(&I);
2787 Value *Agg = I.getAggregateOperand();
2788 Value *AggShadow = DFSF.getShadow(V: Agg);
2789 Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
2790 DFSF.setShadow(I: &I, Shadow: ResShadow);
2791 visitInstOperandOrigins(I);
2792}
2793
2794void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
2795 IRBuilder<> IRB(&I);
2796 Value *AggShadow = DFSF.getShadow(V: I.getAggregateOperand());
2797 Value *InsShadow = DFSF.getShadow(V: I.getInsertedValueOperand());
2798 Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
2799 DFSF.setShadow(I: &I, Shadow: Res);
2800 visitInstOperandOrigins(I);
2801}
2802
2803void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
2804 bool AllLoadsStores = true;
2805 for (User *U : I.users()) {
2806 if (isa<LoadInst>(Val: U))
2807 continue;
2808
2809 if (StoreInst *SI = dyn_cast<StoreInst>(Val: U)) {
2810 if (SI->getPointerOperand() == &I)
2811 continue;
2812 }
2813
2814 AllLoadsStores = false;
2815 break;
2816 }
2817 if (AllLoadsStores) {
2818 IRBuilder<> IRB(&I);
2819 DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(Ty: DFSF.DFS.PrimitiveShadowTy);
2820 if (DFSF.DFS.shouldTrackOrigins()) {
2821 DFSF.AllocaOriginMap[&I] =
2822 IRB.CreateAlloca(Ty: DFSF.DFS.OriginTy, ArraySize: nullptr, Name: "_dfsa");
2823 }
2824 }
2825 DFSF.setShadow(I: &I, Shadow: DFSF.DFS.ZeroPrimitiveShadow);
2826 DFSF.setOrigin(I: &I, Origin: DFSF.DFS.ZeroOrigin);
2827}
2828
2829void DFSanVisitor::visitSelectInst(SelectInst &I) {
2830 Value *CondShadow = DFSF.getShadow(V: I.getCondition());
2831 Value *TrueShadow = DFSF.getShadow(V: I.getTrueValue());
2832 Value *FalseShadow = DFSF.getShadow(V: I.getFalseValue());
2833 Value *ShadowSel = nullptr;
2834 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2835 std::vector<Value *> Shadows;
2836 std::vector<Value *> Origins;
2837 Value *TrueOrigin =
2838 ShouldTrackOrigins ? DFSF.getOrigin(V: I.getTrueValue()) : nullptr;
2839 Value *FalseOrigin =
2840 ShouldTrackOrigins ? DFSF.getOrigin(V: I.getFalseValue()) : nullptr;
2841
2842 DFSF.addConditionalCallbacksIfEnabled(I, Condition: I.getCondition());
2843
2844 if (isa<VectorType>(Val: I.getCondition()->getType())) {
2845 ShadowSel = DFSF.combineShadowsThenConvert(T: I.getType(), V1: TrueShadow,
2846 V2: FalseShadow, Pos: I.getIterator());
2847 if (ShouldTrackOrigins) {
2848 Shadows.push_back(x: TrueShadow);
2849 Shadows.push_back(x: FalseShadow);
2850 Origins.push_back(x: TrueOrigin);
2851 Origins.push_back(x: FalseOrigin);
2852 }
2853 } else {
2854 if (TrueShadow == FalseShadow) {
2855 ShadowSel = TrueShadow;
2856 if (ShouldTrackOrigins) {
2857 Shadows.push_back(x: TrueShadow);
2858 Origins.push_back(x: TrueOrigin);
2859 }
2860 } else {
2861 ShadowSel = SelectInst::Create(C: I.getCondition(), S1: TrueShadow, S2: FalseShadow,
2862 NameStr: "", InsertBefore: I.getIterator());
2863 if (ShouldTrackOrigins) {
2864 Shadows.push_back(x: ShadowSel);
2865 Origins.push_back(x: SelectInst::Create(C: I.getCondition(), S1: TrueOrigin,
2866 S2: FalseOrigin, NameStr: "", InsertBefore: I.getIterator()));
2867 }
2868 }
2869 }
2870 DFSF.setShadow(I: &I, Shadow: DFSF.Opts.dfsan_track_select_control_flow
2871 ? DFSF.combineShadowsThenConvert(T: I.getType(),
2872 V1: CondShadow, V2: ShadowSel,
2873 Pos: I.getIterator())
2874 : ShadowSel);
2875 if (ShouldTrackOrigins) {
2876 if (DFSF.Opts.dfsan_track_select_control_flow) {
2877 Shadows.push_back(x: CondShadow);
2878 Origins.push_back(x: DFSF.getOrigin(V: I.getCondition()));
2879 }
2880 DFSF.setOrigin(I: &I, Origin: DFSF.combineOrigins(Shadows, Origins, Pos: I.getIterator()));
2881 }
2882}
2883
2884void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
2885 IRBuilder<> IRB(&I);
2886 Value *ValShadow = DFSF.getShadow(V: I.getValue());
2887 Value *ValOrigin = DFSF.DFS.shouldTrackOrigins()
2888 ? DFSF.getOrigin(V: I.getValue())
2889 : DFSF.DFS.ZeroOrigin;
2890 IRB.CreateCall(Callee: DFSF.DFS.DFSanSetLabelFn,
2891 Args: {ValShadow, ValOrigin, I.getDest(),
2892 IRB.CreateZExtOrTrunc(V: I.getLength(), DestTy: DFSF.DFS.IntptrTy)});
2893}
2894
2895void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
2896 IRBuilder<> IRB(&I);
2897
2898 // CopyOrMoveOrigin transfers origins by refering to their shadows. So we
2899 // need to move origins before moving shadows.
2900 if (DFSF.DFS.shouldTrackOrigins()) {
2901 IRB.CreateCall(
2902 Callee: DFSF.DFS.DFSanMemOriginTransferFn,
2903 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
2904 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
2905 }
2906
2907 Value *DestShadow = DFSF.DFS.getShadowAddress(Addr: I.getDest(), Pos: I.getIterator());
2908 Value *SrcShadow = DFSF.DFS.getShadowAddress(Addr: I.getSource(), Pos: I.getIterator());
2909 Value *LenShadow =
2910 IRB.CreateMul(LHS: I.getLength(), RHS: ConstantInt::get(Ty: I.getLength()->getType(),
2911 V: DFSF.DFS.ShadowWidthBytes));
2912 auto *MTI = cast<MemTransferInst>(
2913 Val: IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
2914 Args: {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
2915 MTI->setDestAlignment(DFSF.getShadowAlign(InstAlignment: I.getDestAlign().valueOrOne()));
2916 MTI->setSourceAlignment(DFSF.getShadowAlign(InstAlignment: I.getSourceAlign().valueOrOne()));
2917 if (DFSF.Opts.dfsan_event_callbacks) {
2918 IRB.CreateCall(
2919 Callee: DFSF.DFS.DFSanMemTransferCallbackFn,
2920 Args: {DestShadow, IRB.CreateZExtOrTrunc(V: I.getLength(), DestTy: DFSF.DFS.IntptrTy)});
2921 }
2922}
2923
2924void DFSanVisitor::visitCondBrInst(CondBrInst &BR) {
2925 DFSF.addConditionalCallbacksIfEnabled(I&: BR, Condition: BR.getCondition());
2926}
2927
2928void DFSanVisitor::visitSwitchInst(SwitchInst &SW) {
2929 DFSF.addConditionalCallbacksIfEnabled(I&: SW, Condition: SW.getCondition());
2930}
2931
2932static bool isAMustTailRetVal(Value *RetVal) {
2933 // Tail call may have a bitcast between return.
2934 if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
2935 RetVal = I->getOperand(i_nocapture: 0);
2936 }
2937 if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
2938 return I->isMustTailCall();
2939 }
2940 return false;
2941}
2942
2943void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
2944 if (!DFSF.IsNativeABI && RI.getReturnValue()) {
2945 // Don't emit the instrumentation for musttail call returns.
2946 if (isAMustTailRetVal(RetVal: RI.getReturnValue()))
2947 return;
2948
2949 Value *S = DFSF.getShadow(V: RI.getReturnValue());
2950 IRBuilder<> IRB(&RI);
2951 Type *RT = DFSF.F->getFunctionType()->getReturnType();
2952 unsigned Size = getDataLayout().getTypeAllocSize(Ty: DFSF.DFS.getShadowTy(OrigTy: RT));
2953 if (Size <= RetvalTLSSize) {
2954 // If the size overflows, stores nothing. At callsite, oversized return
2955 // shadows are set to zero.
2956 IRB.CreateAlignedStore(Val: S, Ptr: DFSF.getRetvalTLS(T: RT, IRB), Align: ShadowTLSAlignment);
2957 }
2958 if (DFSF.DFS.shouldTrackOrigins()) {
2959 Value *O = DFSF.getOrigin(V: RI.getReturnValue());
2960 IRB.CreateStore(Val: O, Ptr: DFSF.getRetvalOriginTLS());
2961 }
2962 }
2963}
2964
2965void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB,
2966 std::vector<Value *> &Args,
2967 IRBuilder<> &IRB) {
2968 FunctionType *FT = F.getFunctionType();
2969
2970 auto *I = CB.arg_begin();
2971
2972 // Adds non-variable argument shadows.
2973 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
2974 Args.push_back(
2975 x: DFSF.collapseToPrimitiveShadow(Shadow: DFSF.getShadow(V: *I), Pos: CB.getIterator()));
2976
2977 // Adds variable argument shadows.
2978 if (FT->isVarArg()) {
2979 auto *LabelVATy = ArrayType::get(ElementType: DFSF.DFS.PrimitiveShadowTy,
2980 NumElements: CB.arg_size() - FT->getNumParams());
2981 auto *LabelVAAlloca =
2982 new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(),
2983 "labelva", DFSF.F->getEntryBlock().begin());
2984
2985 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
2986 auto *LabelVAPtr = IRB.CreateStructGEP(Ty: LabelVATy, Ptr: LabelVAAlloca, Idx: N);
2987 IRB.CreateStore(
2988 Val: DFSF.collapseToPrimitiveShadow(Shadow: DFSF.getShadow(V: *I), Pos: CB.getIterator()),
2989 Ptr: LabelVAPtr);
2990 }
2991
2992 Args.push_back(x: LabelVAAlloca);
2993 }
2994
2995 // Adds the return value shadow.
2996 if (!FT->getReturnType()->isVoidTy()) {
2997 if (!DFSF.LabelReturnAlloca) {
2998 DFSF.LabelReturnAlloca = new AllocaInst(
2999 DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(),
3000 "labelreturn", DFSF.F->getEntryBlock().begin());
3001 }
3002 Args.push_back(x: DFSF.LabelReturnAlloca);
3003 }
3004}
3005
3006void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB,
3007 std::vector<Value *> &Args,
3008 IRBuilder<> &IRB) {
3009 FunctionType *FT = F.getFunctionType();
3010
3011 auto *I = CB.arg_begin();
3012
3013 // Add non-variable argument origins.
3014 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3015 Args.push_back(x: DFSF.getOrigin(V: *I));
3016
3017 // Add variable argument origins.
3018 if (FT->isVarArg()) {
3019 auto *OriginVATy =
3020 ArrayType::get(ElementType: DFSF.DFS.OriginTy, NumElements: CB.arg_size() - FT->getNumParams());
3021 auto *OriginVAAlloca =
3022 new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(),
3023 "originva", DFSF.F->getEntryBlock().begin());
3024
3025 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3026 auto *OriginVAPtr = IRB.CreateStructGEP(Ty: OriginVATy, Ptr: OriginVAAlloca, Idx: N);
3027 IRB.CreateStore(Val: DFSF.getOrigin(V: *I), Ptr: OriginVAPtr);
3028 }
3029
3030 Args.push_back(x: OriginVAAlloca);
3031 }
3032
3033 // Add the return value origin.
3034 if (!FT->getReturnType()->isVoidTy()) {
3035 if (!DFSF.OriginReturnAlloca) {
3036 DFSF.OriginReturnAlloca = new AllocaInst(
3037 DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(),
3038 "originreturn", DFSF.F->getEntryBlock().begin());
3039 }
3040 Args.push_back(x: DFSF.OriginReturnAlloca);
3041 }
3042}
3043
3044bool DFSanVisitor::visitWrappedCallBase(Function &F, CallBase &CB) {
3045 IRBuilder<> IRB(&CB);
3046 IRB.disableGEPCanonicalization();
3047 switch (DFSF.DFS.getWrapperKind(F: &F)) {
3048 case DataFlowSanitizer::WK_Warning:
3049 CB.setCalledFunction(&F);
3050 IRB.CreateCall(Callee: DFSF.DFS.DFSanUnimplementedFn,
3051 Args: IRB.CreateGlobalString(Str: F.getName()));
3052 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, F: &F);
3053 DFSF.setShadow(I: &CB, Shadow: DFSF.DFS.getZeroShadow(V: &CB));
3054 DFSF.setOrigin(I: &CB, Origin: DFSF.DFS.ZeroOrigin);
3055 return true;
3056 case DataFlowSanitizer::WK_Discard:
3057 CB.setCalledFunction(&F);
3058 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, F: &F);
3059 DFSF.setShadow(I: &CB, Shadow: DFSF.DFS.getZeroShadow(V: &CB));
3060 DFSF.setOrigin(I: &CB, Origin: DFSF.DFS.ZeroOrigin);
3061 return true;
3062 case DataFlowSanitizer::WK_Functional:
3063 CB.setCalledFunction(&F);
3064 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, F: &F);
3065 visitInstOperands(I&: CB);
3066 return true;
3067 case DataFlowSanitizer::WK_Custom:
3068 // Don't try to handle invokes of custom functions, it's too complicated.
3069 // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_
3070 // wrapper.
3071 CallInst *CI = dyn_cast<CallInst>(Val: &CB);
3072 if (!CI)
3073 return false;
3074
3075 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3076 FunctionType *FT = F.getFunctionType();
3077 TransformedFunction CustomFnTy =
3078 DFSF.DFS.getCustomFunctionType(T: FT, TLI&: DFSF.TLI);
3079 std::string CustomFName = ShouldTrackOrigins ? "__dfso_" : "__dfsw_";
3080 CustomFName += F.getName();
3081 FunctionCallee CustomFunCallee = DFSF.DFS.Mod->getOrInsertFunction(
3082 Name: CustomFName, T: CustomFnTy.TransformedType);
3083 if (Function *CustomFun = dyn_cast<Function>(Val: CustomFunCallee.getCallee())) {
3084 // Strange things may occur here: F may have two i64 arguments while
3085 // getOrInsertFunction() returns a preexisting Function with those
3086 // (first) two args as i8:s. Make sure the extensions of those i8:s
3087 // survive copyAttributesFrom() and also add the extensions for the new
3088 // parameters.
3089 AttributeList CustomAL = CustomFun->getAttributes();
3090 CustomFun->copyAttributesFrom(Src: &F);
3091 CustomFun->setAttributes(AttributeList::get(
3092 C&: CI->getContext(),
3093 Attrs: {CustomFun->getAttributes(), CustomAL, CustomFnTy.NewParamAttrs}));
3094
3095 // Custom functions returning non-void will write to the return label.
3096 if (!FT->getReturnType()->isVoidTy()) {
3097 CustomFun->removeFnAttrs(Attrs: DFSF.DFS.ReadOnlyNoneAttrs);
3098 }
3099 }
3100
3101 std::vector<Value *> Args;
3102
3103 // Adds non-variable arguments.
3104 auto *I = CB.arg_begin();
3105 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N) {
3106 Args.push_back(x: *I);
3107 }
3108
3109 // Adds shadow arguments.
3110 addShadowArguments(F, CB, Args, IRB);
3111
3112 // Adds origin arguments.
3113 if (ShouldTrackOrigins)
3114 addOriginArguments(F, CB, Args, IRB);
3115
3116 // Adds variable arguments.
3117 append_range(C&: Args, R: drop_begin(RangeOrContainer: CB.args(), N: FT->getNumParams()));
3118
3119 CallInst *CustomCI = IRB.CreateCall(Callee: CustomFunCallee, Args);
3120 CustomCI->setCallingConv(CI->getCallingConv());
3121 // Add attributes to the parameters from the original call and Function
3122 // and as well those needed for the new parameters.
3123 CustomCI->setAttributes(AttributeList::get(
3124 C&: CI->getContext(),
3125 Attrs: {transformFunctionAttributes(TransformedFunction: CustomFnTy, Ctx&: CI->getContext(),
3126 CallSiteAttrs: CI->getAttributes()),
3127 F.getAttributes(), CustomFnTy.NewParamAttrs}));
3128
3129 // Loads the return value shadow and origin.
3130 if (!FT->getReturnType()->isVoidTy()) {
3131 LoadInst *LabelLoad =
3132 IRB.CreateLoad(Ty: DFSF.DFS.PrimitiveShadowTy, Ptr: DFSF.LabelReturnAlloca);
3133 DFSF.setShadow(I: CustomCI,
3134 Shadow: DFSF.expandFromPrimitiveShadow(
3135 T: FT->getReturnType(), PrimitiveShadow: LabelLoad, Pos: CB.getIterator()));
3136 if (ShouldTrackOrigins) {
3137 LoadInst *OriginLoad =
3138 IRB.CreateLoad(Ty: DFSF.DFS.OriginTy, Ptr: DFSF.OriginReturnAlloca);
3139 DFSF.setOrigin(I: CustomCI, Origin: OriginLoad);
3140 }
3141 }
3142
3143 CI->replaceAllUsesWith(V: CustomCI);
3144 CI->eraseFromParent();
3145 return true;
3146 }
3147 return false;
3148}
3149
3150Value *DFSanVisitor::makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
3151 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3152 uint32_t OrderingTable[NumOrderings] = {};
3153
3154 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3155 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3156 OrderingTable[(int)AtomicOrderingCABI::consume] =
3157 (int)AtomicOrderingCABI::acquire;
3158 OrderingTable[(int)AtomicOrderingCABI::release] =
3159 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3160 (int)AtomicOrderingCABI::acq_rel;
3161 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3162 (int)AtomicOrderingCABI::seq_cst;
3163
3164 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
3165}
3166
3167void DFSanVisitor::visitLibAtomicLoad(CallBase &CB) {
3168 // Since we use getNextNode here, we can't have CB terminate the BB.
3169 assert(isa<CallInst>(CB));
3170
3171 IRBuilder<> IRB(&CB);
3172 Value *Size = CB.getArgOperand(i: 0);
3173 Value *SrcPtr = CB.getArgOperand(i: 1);
3174 Value *DstPtr = CB.getArgOperand(i: 2);
3175 Value *Ordering = CB.getArgOperand(i: 3);
3176 // Convert the call to have at least Acquire ordering to make sure
3177 // the shadow operations aren't reordered before it.
3178 Value *NewOrdering =
3179 IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
3180 CB.setArgOperand(i: 3, v: NewOrdering);
3181
3182 IRBuilder<> NextIRB(CB.getNextNode());
3183 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3184
3185 // TODO: Support -dfsan-combine-pointer-labels-on-load
3186 // TODO: Support -dfsan-event-callbacks
3187
3188 NextIRB.CreateCall(
3189 Callee: DFSF.DFS.DFSanMemShadowOriginTransferFn,
3190 Args: {DstPtr, SrcPtr, NextIRB.CreateIntCast(V: Size, DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
3191}
3192
3193Value *DFSanVisitor::makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
3194 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3195 uint32_t OrderingTable[NumOrderings] = {};
3196
3197 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3198 OrderingTable[(int)AtomicOrderingCABI::release] =
3199 (int)AtomicOrderingCABI::release;
3200 OrderingTable[(int)AtomicOrderingCABI::consume] =
3201 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3202 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3203 (int)AtomicOrderingCABI::acq_rel;
3204 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3205 (int)AtomicOrderingCABI::seq_cst;
3206
3207 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
3208}
3209
3210void DFSanVisitor::visitLibAtomicStore(CallBase &CB) {
3211 IRBuilder<> IRB(&CB);
3212 Value *Size = CB.getArgOperand(i: 0);
3213 Value *SrcPtr = CB.getArgOperand(i: 1);
3214 Value *DstPtr = CB.getArgOperand(i: 2);
3215 Value *Ordering = CB.getArgOperand(i: 3);
3216 // Convert the call to have at least Release ordering to make sure
3217 // the shadow operations aren't reordered after it.
3218 Value *NewOrdering =
3219 IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
3220 CB.setArgOperand(i: 3, v: NewOrdering);
3221
3222 // TODO: Support -dfsan-combine-pointer-labels-on-store
3223 // TODO: Support -dfsan-event-callbacks
3224
3225 IRB.CreateCall(
3226 Callee: DFSF.DFS.DFSanMemShadowOriginTransferFn,
3227 Args: {DstPtr, SrcPtr, IRB.CreateIntCast(V: Size, DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
3228}
3229
3230void DFSanVisitor::visitLibAtomicExchange(CallBase &CB) {
3231 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret, int
3232 // ordering)
3233 IRBuilder<> IRB(&CB);
3234 Value *Size = CB.getArgOperand(i: 0);
3235 Value *TargetPtr = CB.getArgOperand(i: 1);
3236 Value *SrcPtr = CB.getArgOperand(i: 2);
3237 Value *DstPtr = CB.getArgOperand(i: 3);
3238
3239 // This operation is not atomic for the shadow and origin memory.
3240 // This could result in DFSan false positives or false negatives.
3241 // For now we will assume these operations are rare, and
3242 // the additional complexity to address this is not warrented.
3243
3244 // Current Target to Dest
3245 IRB.CreateCall(
3246 Callee: DFSF.DFS.DFSanMemShadowOriginTransferFn,
3247 Args: {DstPtr, TargetPtr, IRB.CreateIntCast(V: Size, DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
3248
3249 // Current Src to Target (overriding)
3250 IRB.CreateCall(
3251 Callee: DFSF.DFS.DFSanMemShadowOriginTransferFn,
3252 Args: {TargetPtr, SrcPtr, IRB.CreateIntCast(V: Size, DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
3253}
3254
3255void DFSanVisitor::visitLibAtomicCompareExchange(CallBase &CB) {
3256 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected, void
3257 // *desired, int success_order, int failure_order)
3258 Value *Size = CB.getArgOperand(i: 0);
3259 Value *TargetPtr = CB.getArgOperand(i: 1);
3260 Value *ExpectedPtr = CB.getArgOperand(i: 2);
3261 Value *DesiredPtr = CB.getArgOperand(i: 3);
3262
3263 // This operation is not atomic for the shadow and origin memory.
3264 // This could result in DFSan false positives or false negatives.
3265 // For now we will assume these operations are rare, and
3266 // the additional complexity to address this is not warrented.
3267
3268 IRBuilder<> NextIRB(CB.getNextNode());
3269 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3270
3271 DFSF.setShadow(I: &CB, Shadow: DFSF.DFS.getZeroShadow(V: &CB));
3272
3273 // If original call returned true, copy Desired to Target.
3274 // If original call returned false, copy Target to Expected.
3275 CallInst *CI = NextIRB.CreateCall(
3276 Callee: DFSF.DFS.DFSanMemShadowOriginConditionalExchangeFn,
3277 Args: {NextIRB.CreateIntCast(V: &CB, DestTy: NextIRB.getInt8Ty(), isSigned: false), TargetPtr,
3278 ExpectedPtr, DesiredPtr,
3279 NextIRB.CreateIntCast(V: Size, DestTy: DFSF.DFS.IntptrTy, isSigned: false)});
3280 CI->maybeAddParamAttr(ArgNo: 0, Kind: DFSF.TLI.getExtAttrForI8Param(/*Signed=*/false));
3281}
3282
3283void DFSanVisitor::visitCallBase(CallBase &CB) {
3284 Function *F = CB.getCalledFunction();
3285 if ((F && F->isIntrinsic()) || CB.isInlineAsm()) {
3286 visitInstOperands(I&: CB);
3287 return;
3288 }
3289
3290 // Calls to this function are synthesized in wrappers, and we shouldn't
3291 // instrument them.
3292 if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
3293 return;
3294
3295 LibFunc LF = DFSF.TLI.getLibFunc(CB);
3296 if (LF != NotLibFunc) {
3297 // libatomic.a functions need to have special handling because there isn't
3298 // a good way to intercept them or compile the library with
3299 // instrumentation.
3300 switch (LF) {
3301 case LibFunc_atomic_load:
3302 if (!isa<CallInst>(Val: CB)) {
3303 llvm::errs() << "DFSAN -- cannot instrument invoke of libatomic load. "
3304 "Ignoring!\n";
3305 break;
3306 }
3307 visitLibAtomicLoad(CB);
3308 return;
3309 case LibFunc_atomic_store:
3310 visitLibAtomicStore(CB);
3311 return;
3312 default:
3313 break;
3314 }
3315 }
3316
3317 // TODO: These are not supported by TLI? They are not in the enum.
3318 if (F && F->hasName() && !F->isVarArg()) {
3319 if (F->getName() == "__atomic_exchange") {
3320 visitLibAtomicExchange(CB);
3321 return;
3322 }
3323 if (F->getName() == "__atomic_compare_exchange") {
3324 visitLibAtomicCompareExchange(CB);
3325 return;
3326 }
3327 }
3328
3329 auto UnwrappedFnIt = DFSF.DFS.UnwrappedFnMap.find(Val: CB.getCalledOperand());
3330 if (UnwrappedFnIt != DFSF.DFS.UnwrappedFnMap.end())
3331 if (visitWrappedCallBase(F&: *UnwrappedFnIt->second, CB))
3332 return;
3333
3334 IRBuilder<> IRB(&CB);
3335
3336 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3337 FunctionType *FT = CB.getFunctionType();
3338 const DataLayout &DL = getDataLayout();
3339
3340 // Stores argument shadows.
3341 unsigned ArgOffset = 0;
3342 for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
3343 if (ShouldTrackOrigins) {
3344 // Ignore overflowed origins
3345 Value *ArgShadow = DFSF.getShadow(V: CB.getArgOperand(i: I));
3346 if (I < DFSF.DFS.NumOfElementsInArgOrgTLS &&
3347 !DFSF.DFS.isZeroShadow(V: ArgShadow))
3348 IRB.CreateStore(Val: DFSF.getOrigin(V: CB.getArgOperand(i: I)),
3349 Ptr: DFSF.getArgOriginTLS(ArgNo: I, IRB));
3350 }
3351
3352 unsigned Size =
3353 DL.getTypeAllocSize(Ty: DFSF.DFS.getShadowTy(OrigTy: FT->getParamType(i: I)));
3354 // Stop storing if arguments' size overflows. Inside a function, arguments
3355 // after overflow have zero shadow values.
3356 if (ArgOffset + Size > ArgTLSSize)
3357 break;
3358 IRB.CreateAlignedStore(Val: DFSF.getShadow(V: CB.getArgOperand(i: I)),
3359 Ptr: DFSF.getArgTLS(T: FT->getParamType(i: I), ArgOffset, IRB),
3360 Align: ShadowTLSAlignment);
3361 ArgOffset += alignTo(Size, A: ShadowTLSAlignment);
3362 }
3363
3364 Instruction *Next = nullptr;
3365 if (!CB.getType()->isVoidTy()) {
3366 if (InvokeInst *II = dyn_cast<InvokeInst>(Val: &CB)) {
3367 if (II->getNormalDest()->getSinglePredecessor()) {
3368 Next = &II->getNormalDest()->front();
3369 } else {
3370 BasicBlock *NewBB =
3371 SplitEdge(From: II->getParent(), To: II->getNormalDest(), DT: &DFSF.DT);
3372 Next = &NewBB->front();
3373 }
3374 } else {
3375 assert(CB.getIterator() != CB.getParent()->end());
3376 Next = CB.getNextNode();
3377 }
3378
3379 // Don't emit the epilogue for musttail call returns.
3380 if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
3381 return;
3382
3383 // Loads the return value shadow.
3384 IRBuilder<> NextIRB(Next);
3385 unsigned Size = DL.getTypeAllocSize(Ty: DFSF.DFS.getShadowTy(V: &CB));
3386 if (Size > RetvalTLSSize) {
3387 // Set overflowed return shadow to be zero.
3388 DFSF.setShadow(I: &CB, Shadow: DFSF.DFS.getZeroShadow(V: &CB));
3389 } else {
3390 LoadInst *LI = NextIRB.CreateAlignedLoad(
3391 Ty: DFSF.DFS.getShadowTy(V: &CB), Ptr: DFSF.getRetvalTLS(T: CB.getType(), IRB&: NextIRB),
3392 Align: ShadowTLSAlignment, Name: "_dfsret");
3393 DFSF.SkipInsts.insert(V: LI);
3394 DFSF.setShadow(I: &CB, Shadow: LI);
3395 DFSF.NonZeroChecks.push_back(x: LI);
3396 }
3397
3398 if (ShouldTrackOrigins) {
3399 LoadInst *LI = NextIRB.CreateLoad(Ty: DFSF.DFS.OriginTy,
3400 Ptr: DFSF.getRetvalOriginTLS(), Name: "_dfsret_o");
3401 DFSF.SkipInsts.insert(V: LI);
3402 DFSF.setOrigin(I: &CB, Origin: LI);
3403 }
3404
3405 DFSF.addReachesFunctionCallbacksIfEnabled(IRB&: NextIRB, I&: CB, Data: &CB);
3406 }
3407}
3408
3409void DFSanVisitor::visitPHINode(PHINode &PN) {
3410 Type *ShadowTy = DFSF.DFS.getShadowTy(V: &PN);
3411 PHINode *ShadowPN = PHINode::Create(Ty: ShadowTy, NumReservedValues: PN.getNumIncomingValues(), NameStr: "",
3412 InsertBefore: PN.getIterator());
3413
3414 // Give the shadow phi node valid predecessors to fool SplitEdge into working.
3415 Value *PoisonShadow = PoisonValue::get(T: ShadowTy);
3416 for (BasicBlock *BB : PN.blocks())
3417 ShadowPN->addIncoming(V: PoisonShadow, BB);
3418
3419 DFSF.setShadow(I: &PN, Shadow: ShadowPN);
3420
3421 PHINode *OriginPN = nullptr;
3422 if (DFSF.DFS.shouldTrackOrigins()) {
3423 OriginPN = PHINode::Create(Ty: DFSF.DFS.OriginTy, NumReservedValues: PN.getNumIncomingValues(), NameStr: "",
3424 InsertBefore: PN.getIterator());
3425 Value *PoisonOrigin = PoisonValue::get(T: DFSF.DFS.OriginTy);
3426 for (BasicBlock *BB : PN.blocks())
3427 OriginPN->addIncoming(V: PoisonOrigin, BB);
3428 DFSF.setOrigin(I: &PN, Origin: OriginPN);
3429 }
3430
3431 DFSF.PHIFixups.push_back(x: {.Phi: &PN, .ShadowPhi: ShadowPN, .OriginPhi: OriginPN});
3432}
3433
3434PreservedAnalyses DataFlowSanitizerPass::run(Module &M,
3435 ModuleAnalysisManager &AM) {
3436 // Return early if nosanitize_dataflow module flag is present for the module.
3437 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_dataflow"))
3438 return PreservedAnalyses::all();
3439 auto GetTLI = [&](Function &F) -> TargetLibraryInfo & {
3440 auto &FAM =
3441 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
3442 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
3443 };
3444 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
3445 if (!DataFlowSanitizer(Opts, ABIListFiles, FS).runImpl(M, GetTLI))
3446 return PreservedAnalyses::all();
3447
3448 PreservedAnalyses PA = PreservedAnalyses::none();
3449 // GlobalsAA is considered stateless and does not get invalidated unless
3450 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
3451 // make changes that require GlobalsAA to be invalidated.
3452 PA.abandon<GlobalsAA>();
3453 return PA;
3454}
3455