1//===- HWAddressSanitizer.cpp - memory access error detector --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of HWAddressSanitizer, an address basic correctness
11/// checker based on tagged addressing.
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15#include "llvm/ADT/MapVector.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/DomTreeUpdater.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ProfileSummaryInfo.h"
27#include "llvm/Analysis/StackSafetyAnalysis.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/BinaryFormat/Dwarf.h"
31#include "llvm/BinaryFormat/ELF.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/Dominators.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/InstIterator.h"
43#include "llvm/IR/Instruction.h"
44#include "llvm/IR/Instructions.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/LLVMContext.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/IR/Module.h"
50#include "llvm/IR/Type.h"
51#include "llvm/IR/Value.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Debug.h"
55#include "llvm/Support/MD5.h"
56#include "llvm/Support/RandomNumberGenerator.h"
57#include "llvm/Support/raw_ostream.h"
58#include "llvm/TargetParser/Triple.h"
59#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
60#include "llvm/Transforms/Utils/BasicBlockUtils.h"
61#include "llvm/Transforms/Utils/Instrumentation.h"
62#include "llvm/Transforms/Utils/Local.h"
63#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
64#include "llvm/Transforms/Utils/ModuleUtils.h"
65#include "llvm/Transforms/Utils/PromoteMemToReg.h"
66#include <optional>
67#include <random>
68
69using namespace llvm;
70
71#define DEBUG_TYPE "hwasan"
72
73const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
74const char kHwasanNoteName[] = "hwasan.note";
75const char kHwasanInitName[] = "__hwasan_init";
76const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
77
78const char kHwasanShadowMemoryDynamicAddress[] =
79 "__hwasan_shadow_memory_dynamic_address";
80
81// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
82static const size_t kNumberOfAccessSizes = 5;
83
84static const size_t kDefaultShadowScale = 4;
85
86static const unsigned kShadowBaseAlignment = 32;
87
88namespace {
89enum class OffsetKind {
90 kFixed = 0,
91 kGlobal,
92 kIfunc,
93 kTls,
94};
95}
96
97static cl::opt<std::string>
98 ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
99 cl::desc("Prefix for memory access callbacks"),
100 cl::Hidden, cl::init(Val: "__hwasan_"));
101
102static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
103 "hwasan-kernel-mem-intrinsic-prefix",
104 cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
105 cl::init(Val: false));
106
107static cl::opt<bool> ClInstrumentWithCalls(
108 "hwasan-instrument-with-calls",
109 cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
110 cl::init(Val: false));
111
112static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
113 cl::desc("instrument read instructions"),
114 cl::Hidden, cl::init(Val: true));
115
116static cl::opt<bool>
117 ClInstrumentWrites("hwasan-instrument-writes",
118 cl::desc("instrument write instructions"), cl::Hidden,
119 cl::init(Val: true));
120
121static cl::opt<bool> ClInstrumentAtomics(
122 "hwasan-instrument-atomics",
123 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
124 cl::init(Val: true));
125
126static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
127 cl::desc("instrument byval arguments"),
128 cl::Hidden, cl::init(Val: true));
129
130static cl::opt<bool>
131 ClRecover("hwasan-recover",
132 cl::desc("Enable recovery mode (continue-after-error)."),
133 cl::Hidden, cl::init(Val: false));
134
135static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
136 cl::desc("instrument stack (allocas)"),
137 cl::Hidden, cl::init(Val: true));
138
139static cl::opt<bool>
140 ClUseStackSafety("hwasan-use-stack-safety", cl::Hidden, cl::init(Val: true),
141 cl::Hidden, cl::desc("Use Stack Safety analysis results"),
142 cl::Optional);
143
144static cl::opt<size_t> ClMaxLifetimes(
145 "hwasan-max-lifetimes-for-alloca", cl::Hidden, cl::init(Val: 3),
146 cl::ReallyHidden,
147 cl::desc("How many lifetime ends to handle for a single alloca."),
148 cl::Optional);
149
150static cl::opt<bool>
151 ClUseAfterScope("hwasan-use-after-scope",
152 cl::desc("detect use after scope within function"),
153 cl::Hidden, cl::init(Val: true));
154
155static cl::opt<bool> ClStrictUseAfterScope(
156 "hwasan-strict-use-after-scope",
157 cl::desc("for complicated lifetimes, tag both on end and return"),
158 cl::Hidden, cl::init(Val: true));
159
160static cl::opt<bool> ClGenerateTagsWithCalls(
161 "hwasan-generate-tags-with-calls",
162 cl::desc("generate new tags with runtime library calls"), cl::Hidden,
163 cl::init(Val: false));
164
165static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
166 cl::Hidden, cl::init(Val: false));
167
168static cl::opt<bool> ClAllGlobals(
169 "hwasan-all-globals",
170 cl::desc(
171 "Instrument globals, even those within user-defined sections. Warning: "
172 "This may break existing code which walks globals via linker-generated "
173 "symbols, expects certain globals to be contiguous with each other, or "
174 "makes other assumptions which are invalidated by HWASan "
175 "instrumentation."),
176 cl::Hidden, cl::init(Val: false));
177
178static cl::opt<int> ClMatchAllTag(
179 "hwasan-match-all-tag",
180 cl::desc("don't report bad accesses via pointers with this tag"),
181 cl::Hidden, cl::init(Val: -1));
182
183static cl::opt<bool>
184 ClEnableKhwasan("hwasan-kernel",
185 cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
186 cl::Hidden, cl::init(Val: false));
187
188// These flags allow to change the shadow mapping and control how shadow memory
189// is accessed. The shadow mapping looks like:
190// Shadow = (Mem >> scale) + offset
191
192static cl::opt<uint64_t>
193 ClMappingOffset("hwasan-mapping-offset",
194 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
195 cl::Hidden);
196
197static cl::opt<OffsetKind> ClMappingOffsetDynamic(
198 "hwasan-mapping-offset-dynamic",
199 cl::desc("HWASan shadow mapping dynamic offset location"), cl::Hidden,
200 cl::values(clEnumValN(OffsetKind::kGlobal, "global", "Use global"),
201 clEnumValN(OffsetKind::kIfunc, "ifunc", "Use ifunc global"),
202 clEnumValN(OffsetKind::kTls, "tls", "Use TLS")));
203
204static cl::opt<bool>
205 ClFrameRecords("hwasan-with-frame-record",
206 cl::desc("Use ring buffer for stack allocations"),
207 cl::Hidden);
208
209static cl::opt<int> ClHotPercentileCutoff("hwasan-percentile-cutoff-hot",
210 cl::desc("Hot percentile cutoff."));
211
212static cl::opt<float>
213 ClRandomKeepRate("hwasan-random-rate",
214 cl::desc("Probability value in the range [0.0, 1.0] "
215 "to keep instrumentation of a function. "
216 "Note: instrumentation can be skipped randomly "
217 "OR because of the hot percentile cutoff, if "
218 "both are supplied."));
219
220static cl::opt<bool> ClStaticLinking(
221 "hwasan-static-linking",
222 cl::desc("Don't use .note.hwasan.globals section to instrument globals "
223 "from loadable libraries. "
224 "Note: in static binaries, the global variables section can be "
225 "accessed directly via linker-provided "
226 "__start_hwasan_globals and __stop_hwasan_globals symbols"),
227 cl::Hidden, cl::init(Val: false));
228
229STATISTIC(NumTotalFuncs, "Number of total funcs");
230STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs");
231STATISTIC(NumNoProfileSummaryFuncs, "Number of funcs without PS");
232
233// Mode for selecting how to insert frame record info into the stack ring
234// buffer.
235enum RecordStackHistoryMode {
236 // Do not record frame record info.
237 none,
238
239 // Insert instructions into the prologue for storing into the stack ring
240 // buffer directly.
241 instr,
242
243 // Add a call to __hwasan_add_frame_record in the runtime.
244 libcall,
245};
246
247static cl::opt<RecordStackHistoryMode> ClRecordStackHistory(
248 "hwasan-record-stack-history",
249 cl::desc("Record stack frames with tagged allocations in a thread-local "
250 "ring buffer"),
251 cl::values(clEnumVal(none, "Do not record stack ring history"),
252 clEnumVal(instr, "Insert instructions into the prologue for "
253 "storing into the stack ring buffer directly"),
254 clEnumVal(libcall, "Add a call to __hwasan_add_frame_record for "
255 "storing into the stack ring buffer")),
256 cl::Hidden, cl::init(Val: instr));
257
258static cl::opt<bool>
259 ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
260 cl::desc("instrument memory intrinsics"),
261 cl::Hidden, cl::init(Val: true));
262
263static cl::opt<bool>
264 ClInstrumentLandingPads("hwasan-instrument-landing-pads",
265 cl::desc("instrument landing pads"), cl::Hidden,
266 cl::init(Val: false));
267
268static cl::opt<bool> ClUseShortGranules(
269 "hwasan-use-short-granules",
270 cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
271 cl::init(Val: false));
272
273static cl::opt<bool> ClInstrumentPersonalityFunctions(
274 "hwasan-instrument-personality-functions",
275 cl::desc("instrument personality functions"), cl::Hidden);
276
277static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
278 cl::desc("inline all checks"),
279 cl::Hidden, cl::init(Val: false));
280
281static cl::opt<bool> ClInlineFastPathChecks("hwasan-inline-fast-path-checks",
282 cl::desc("inline all checks"),
283 cl::Hidden, cl::init(Val: false));
284
285// Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
286static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
287 cl::desc("Use page aliasing in HWASan"),
288 cl::Hidden, cl::init(Val: false));
289
290namespace {
291
292template <typename T> T optOr(cl::opt<T> &Opt, T Other) {
293 return Opt.getNumOccurrences() ? Opt : Other;
294}
295
296bool shouldUsePageAliases(const Triple &TargetTriple) {
297 return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64;
298}
299
300bool shouldInstrumentStack(const Triple &TargetTriple) {
301 return !shouldUsePageAliases(TargetTriple) && ClInstrumentStack;
302}
303
304bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
305 return optOr(Opt&: ClInstrumentWithCalls, Other: TargetTriple.getArch() == Triple::x86_64);
306}
307
308bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
309 return optOr(Opt&: ClUseStackSafety, Other: !DisableOptimization);
310}
311
312bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple,
313 bool DisableOptimization) {
314 return shouldInstrumentStack(TargetTriple) &&
315 mightUseStackSafetyAnalysis(DisableOptimization);
316}
317
318bool shouldDetectUseAfterScope(const Triple &TargetTriple) {
319 return ClUseAfterScope && shouldInstrumentStack(TargetTriple);
320}
321
322/// An instrumentation pass implementing detection of addressability bugs
323/// using tagged pointers.
324class HWAddressSanitizer {
325public:
326 HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover,
327 const StackSafetyGlobalInfo *SSI)
328 : M(M), SSI(SSI) {
329 this->Recover = optOr(Opt&: ClRecover, Other: Recover);
330 this->CompileKernel = optOr(Opt&: ClEnableKhwasan, Other: CompileKernel);
331 this->Rng = ClRandomKeepRate.getNumOccurrences() ? M.createRNG(DEBUG_TYPE)
332 : nullptr;
333
334 initializeModule();
335 }
336
337 void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
338
339private:
340 struct ShadowTagCheckInfo {
341 Instruction *TagMismatchTerm = nullptr;
342 Value *PtrLong = nullptr;
343 Value *AddrLong = nullptr;
344 Value *PtrTag = nullptr;
345 Value *MemTag = nullptr;
346 };
347
348 bool selectiveInstrumentationShouldSkip(Function &F,
349 FunctionAnalysisManager &FAM) const;
350 void initializeModule();
351 void createHwasanCtorComdat();
352 void createHwasanNote();
353
354 void initializeCallbacks(Module &M);
355
356 Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
357
358 Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
359 Value *getShadowNonTls(IRBuilder<> &IRB);
360
361 void untagPointerOperand(Instruction *I, Value *Addr);
362 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
363
364 int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
365 ShadowTagCheckInfo insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
366 DomTreeUpdater &DTU, LoopInfo *LI);
367 void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
368 unsigned AccessSizeIndex,
369 Instruction *InsertBefore,
370 DomTreeUpdater &DTU, LoopInfo *LI);
371 void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
372 unsigned AccessSizeIndex,
373 Instruction *InsertBefore, DomTreeUpdater &DTU,
374 LoopInfo *LI);
375 bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI);
376 void instrumentMemIntrinsic(MemIntrinsic *MI);
377 bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
378 LoopInfo *LI, const DataLayout &DL);
379 bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr);
380 bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst,
381 Value *Ptr);
382
383 void getInterestingMemoryOperands(
384 OptimizationRemarkEmitter &ORE, Instruction *I,
385 const TargetLibraryInfo &TLI,
386 SmallVectorImpl<InterestingMemoryOperand> &Interesting);
387
388 void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
389 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
390 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
391 void instrumentStack(OptimizationRemarkEmitter &ORE, memtag::StackInfo &Info,
392 Value *StackTag, Value *UARTag, const DominatorTree &DT,
393 const PostDominatorTree &PDT, const LoopInfo &LI);
394 void instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
395 Value *getNextTagWithCall(IRBuilder<> &IRB);
396 Value *getStackBaseTag(IRBuilder<> &IRB);
397 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
398 Value *getUARTag(IRBuilder<> &IRB);
399
400 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB);
401 Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
402 unsigned retagMask(unsigned AllocaNo);
403
404 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
405
406 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
407 void instrumentGlobals();
408
409 Value *getCachedFP(IRBuilder<> &IRB);
410 Value *getFrameRecordInfo(IRBuilder<> &IRB);
411
412 void instrumentPersonalityFunctions();
413
414 LLVMContext *C;
415 Module &M;
416 const StackSafetyGlobalInfo *SSI;
417 Triple TargetTriple;
418 std::unique_ptr<RandomNumberGenerator> Rng;
419
420 /// This struct defines the shadow mapping using the rule:
421 /// If `kFixed`, then
422 /// shadow = (mem >> Scale) + Offset.
423 /// If `kGlobal`, then
424 /// extern char* __hwasan_shadow_memory_dynamic_address;
425 /// shadow = (mem >> Scale) + __hwasan_shadow_memory_dynamic_address
426 /// If `kIfunc`, then
427 /// extern char __hwasan_shadow[];
428 /// shadow = (mem >> Scale) + &__hwasan_shadow
429 /// If `kTls`, then
430 /// extern char *__hwasan_tls;
431 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
432 ///
433 /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
434 /// ring buffer for storing stack allocations on targets that support it.
435 class ShadowMapping {
436 OffsetKind Kind;
437 uint64_t Offset;
438 uint8_t Scale;
439 bool WithFrameRecord;
440
441 void SetFixed(uint64_t O) {
442 Kind = OffsetKind::kFixed;
443 Offset = O;
444 }
445
446 public:
447 void init(Triple &TargetTriple, bool InstrumentWithCalls,
448 bool CompileKernel);
449 Align getObjectAlignment() const { return Align(1ULL << Scale); }
450 bool isInGlobal() const { return Kind == OffsetKind::kGlobal; }
451 bool isInIfunc() const { return Kind == OffsetKind::kIfunc; }
452 bool isInTls() const { return Kind == OffsetKind::kTls; }
453 bool isFixed() const { return Kind == OffsetKind::kFixed; }
454 uint8_t scale() const { return Scale; };
455 uint64_t offset() const {
456 assert(isFixed());
457 return Offset;
458 };
459 bool withFrameRecord() const { return WithFrameRecord; };
460 };
461
462 ShadowMapping Mapping;
463
464 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
465 Type *IntptrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
466 PointerType *PtrTy = PointerType::getUnqual(C&: M.getContext());
467 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
468 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
469 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
470
471 bool CompileKernel;
472 bool Recover;
473 bool OutlinedChecks;
474 bool InlineFastPath;
475 bool UseShortGranules;
476 bool InstrumentLandingPads;
477 bool InstrumentWithCalls;
478 bool InstrumentStack;
479 bool InstrumentGlobals;
480 bool DetectUseAfterScope;
481 bool UsePageAliases;
482 bool UseMatchAllCallback;
483
484 std::optional<uint8_t> MatchAllTag;
485
486 unsigned PointerTagShift;
487 uint64_t TagMaskByte;
488
489 Function *HwasanCtorFunction;
490
491 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
492 FunctionCallee HwasanMemoryAccessCallbackSized[2];
493
494 FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
495 FunctionCallee HwasanHandleVfork;
496
497 FunctionCallee HwasanTagMemoryFunc;
498 FunctionCallee HwasanGenerateTagFunc;
499 FunctionCallee HwasanRecordFrameRecordFunc;
500
501 Constant *ShadowGlobal;
502
503 Value *ShadowBase = nullptr;
504 Value *StackBaseTag = nullptr;
505 Value *CachedFP = nullptr;
506 GlobalValue *ThreadPtrGlobal = nullptr;
507};
508
509} // end anonymous namespace
510
511PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
512 ModuleAnalysisManager &MAM) {
513 // Return early if nosanitize_hwaddress module flag is present for the module.
514 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_hwaddress"))
515 return PreservedAnalyses::all();
516 const StackSafetyGlobalInfo *SSI = nullptr;
517 const Triple &TargetTriple = M.getTargetTriple();
518 if (shouldUseStackSafetyAnalysis(TargetTriple, DisableOptimization: Options.DisableOptimization))
519 SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M);
520
521 HWAddressSanitizer HWASan(M, Options.CompileKernel, Options.Recover, SSI);
522 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
523 for (Function &F : M)
524 HWASan.sanitizeFunction(F, FAM);
525
526 PreservedAnalyses PA = PreservedAnalyses::none();
527 // DominatorTreeAnalysis, PostDominatorTreeAnalysis, and LoopAnalysis
528 // are incrementally updated throughout this pass whenever
529 // SplitBlockAndInsertIfThen is called.
530 PA.preserve<DominatorTreeAnalysis>();
531 PA.preserve<PostDominatorTreeAnalysis>();
532 PA.preserve<LoopAnalysis>();
533 // GlobalsAA is considered stateless and does not get invalidated unless
534 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
535 // make changes that require GlobalsAA to be invalidated.
536 PA.abandon<GlobalsAA>();
537 return PA;
538}
539void HWAddressSanitizerPass::printPipeline(
540 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
541 static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
542 OS, MapClassName2PassName);
543 OS << '<';
544 if (Options.CompileKernel)
545 OS << "kernel;";
546 if (Options.Recover)
547 OS << "recover";
548 OS << '>';
549}
550
551void HWAddressSanitizer::createHwasanNote() {
552 // Create a note that contains pointers to the list of global
553 // descriptors. Adding a note to the output file will cause the linker to
554 // create a PT_NOTE program header pointing to the note that we can use to
555 // find the descriptor list starting from the program headers. A function
556 // provided by the runtime initializes the shadow memory for the globals by
557 // accessing the descriptor list via the note. The dynamic loader needs to
558 // call this function whenever a library is loaded.
559 //
560 // The reason why we use a note for this instead of a more conventional
561 // approach of having a global constructor pass a descriptor list pointer to
562 // the runtime is because of an order of initialization problem. With
563 // constructors we can encounter the following problematic scenario:
564 //
565 // 1) library A depends on library B and also interposes one of B's symbols
566 // 2) B's constructors are called before A's (as required for correctness)
567 // 3) during construction, B accesses one of its "own" globals (actually
568 // interposed by A) and triggers a HWASAN failure due to the initialization
569 // for A not having happened yet
570 //
571 // Even without interposition it is possible to run into similar situations in
572 // cases where two libraries mutually depend on each other.
573 //
574 // We only need one note per binary, so put everything for the note in a
575 // comdat. This needs to be a comdat with an .init_array section to prevent
576 // newer versions of lld from discarding the note.
577 //
578 // Create the note even if we aren't instrumenting globals. This ensures that
579 // binaries linked from object files with both instrumented and
580 // non-instrumented globals will end up with a note, even if a comdat from an
581 // object file with non-instrumented globals is selected. The note is harmless
582 // if the runtime doesn't support it, since it will just be ignored.
583 Comdat *NoteComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
584
585 Type *Int8Arr0Ty = ArrayType::get(ElementType: Int8Ty, NumElements: 0);
586 auto *Start =
587 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
588 nullptr, "__start_hwasan_globals");
589 Start->setVisibility(GlobalValue::HiddenVisibility);
590 auto *Stop =
591 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
592 nullptr, "__stop_hwasan_globals");
593 Stop->setVisibility(GlobalValue::HiddenVisibility);
594
595 // Null-terminated so actually 8 bytes, which are required in order to align
596 // the note properly.
597 auto *Name = ConstantDataArray::get(Context&: *C, Elts: "LLVM\0\0\0");
598
599 auto *NoteTy = StructType::get(elt1: Int32Ty, elts: Int32Ty, elts: Int32Ty, elts: Name->getType(),
600 elts: Int32Ty, elts: Int32Ty);
601 auto *Note =
602 new GlobalVariable(M, NoteTy, /*isConstant=*/true,
603 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
604 Note->setSection(".note.hwasan.globals");
605 Note->setComdat(NoteComdat);
606 Note->setAlignment(Align(4));
607
608 // The pointers in the note need to be relative so that the note ends up being
609 // placed in rodata, which is the standard location for notes.
610 auto CreateRelPtr = [&](Constant *Ptr) {
611 return ConstantExpr::getTrunc(
612 C: ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: Ptr, Ty: Int64Ty),
613 C2: ConstantExpr::getPtrToInt(C: Note, Ty: Int64Ty)),
614 Ty: Int32Ty);
615 };
616 Note->setInitializer(ConstantStruct::getAnon(
617 V: {ConstantInt::get(Ty: Int32Ty, V: 8), // n_namesz
618 ConstantInt::get(Ty: Int32Ty, V: 8), // n_descsz
619 ConstantInt::get(Ty: Int32Ty, V: ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
620 Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
621 appendToCompilerUsed(M, Values: Note);
622
623 // Create a zero-length global in hwasan_globals so that the linker will
624 // always create start and stop symbols.
625 auto *Dummy = new GlobalVariable(
626 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
627 Constant::getNullValue(Ty: Int8Arr0Ty), "hwasan.dummy.global");
628 Dummy->setSection("hwasan_globals");
629 Dummy->setComdat(NoteComdat);
630 Dummy->setMetadata(KindID: LLVMContext::MD_associated,
631 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: Note)));
632 appendToCompilerUsed(M, Values: Dummy);
633}
634
635void HWAddressSanitizer::createHwasanCtorComdat() {
636 std::tie(args&: HwasanCtorFunction, args: std::ignore) =
637 getOrCreateSanitizerCtorAndInitFunctions(
638 M, CtorName: kHwasanModuleCtorName, InitName: kHwasanInitName,
639 /*InitArgTypes=*/{},
640 /*InitArgs=*/{},
641 // This callback is invoked when the functions are created the first
642 // time. Hook them into the global ctors list in that case:
643 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
644 Comdat *CtorComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
645 Ctor->setComdat(CtorComdat);
646 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
647 });
648
649 // Do not create .note.hwasan.globals for static binaries, as it is only
650 // needed for instrumenting globals from dynamic libraries. In static
651 // binaries, the global variables section can be accessed directly via the
652 // __start_hwasan_globals and __stop_hwasan_globals symbols inserted by the
653 // linker.
654 if (!ClStaticLinking)
655 createHwasanNote();
656}
657
658/// Module-level initialization.
659///
660/// inserts a call to __hwasan_init to the module's constructor list.
661void HWAddressSanitizer::initializeModule() {
662 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
663 TargetTriple = M.getTargetTriple();
664
665 // HWASan may do short granule checks on function arguments read from the
666 // argument memory (last byte of the granule), which invalidates writeonly.
667 for (Function &F : M.functions())
668 removeASanIncompatibleFnAttributes(F, /*ReadsArgMem=*/true);
669
670 // x86_64 currently has two modes:
671 // - Intel LAM (default)
672 // - pointer aliasing (heap only)
673 bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
674 UsePageAliases = shouldUsePageAliases(TargetTriple);
675 InstrumentWithCalls = shouldInstrumentWithCalls(TargetTriple);
676 InstrumentStack = shouldInstrumentStack(TargetTriple);
677 DetectUseAfterScope = shouldDetectUseAfterScope(TargetTriple);
678 PointerTagShift = IsX86_64 ? 57 : 56;
679 TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
680
681 Mapping.init(TargetTriple, InstrumentWithCalls, CompileKernel);
682
683 C = &(M.getContext());
684 IRBuilder<> IRB(*C);
685
686 HwasanCtorFunction = nullptr;
687
688 // Older versions of Android do not have the required runtime support for
689 // short granules, global or personality function instrumentation. On other
690 // platforms we currently require using the latest version of the runtime.
691 bool NewRuntime =
692 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(Major: 30);
693
694 UseShortGranules = optOr(Opt&: ClUseShortGranules, Other: NewRuntime);
695 OutlinedChecks = (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
696 TargetTriple.isOSBinFormatELF() &&
697 !optOr(Opt&: ClInlineAllChecks, Other: Recover);
698
699 // These platforms may prefer less inlining to reduce binary size.
700 InlineFastPath = optOr(Opt&: ClInlineFastPathChecks, Other: !(TargetTriple.isAndroid() ||
701 TargetTriple.isOSFuchsia()));
702
703 if (ClMatchAllTag.getNumOccurrences()) {
704 if (ClMatchAllTag != -1) {
705 MatchAllTag = ClMatchAllTag & 0xFF;
706 }
707 } else if (CompileKernel) {
708 MatchAllTag = 0xFF;
709 }
710 UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
711
712 // If we don't have personality function support, fall back to landing pads.
713 InstrumentLandingPads = optOr(Opt&: ClInstrumentLandingPads, Other: !NewRuntime);
714
715 InstrumentGlobals =
716 !CompileKernel && !UsePageAliases && optOr(Opt&: ClGlobals, Other: NewRuntime);
717
718 if (!CompileKernel) {
719 if (InstrumentGlobals)
720 instrumentGlobals();
721
722 createHwasanCtorComdat();
723
724 bool InstrumentPersonalityFunctions =
725 optOr(Opt&: ClInstrumentPersonalityFunctions, Other: NewRuntime);
726 if (InstrumentPersonalityFunctions)
727 instrumentPersonalityFunctions();
728 }
729
730 if (!TargetTriple.isAndroid()) {
731 ThreadPtrGlobal = M.getOrInsertGlobal(Name: "__hwasan_tls", Ty: IntptrTy, CreateGlobalCallback: [&] {
732 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
733 GlobalValue::ExternalLinkage, nullptr,
734 "__hwasan_tls", nullptr,
735 GlobalVariable::InitialExecTLSModel);
736 appendToCompilerUsed(M, Values: GV);
737 return GV;
738 });
739 }
740}
741
742void HWAddressSanitizer::initializeCallbacks(Module &M) {
743 IRBuilder<> IRB(*C);
744 const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
745 FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
746 *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
747 *HwasanMemsetFnTy;
748 if (UseMatchAllCallback) {
749 HwasanMemoryAccessCallbackSizedFnTy =
750 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy, Int8Ty}, isVarArg: false);
751 HwasanMemoryAccessCallbackFnTy =
752 FunctionType::get(Result: VoidTy, Params: {IntptrTy, Int8Ty}, isVarArg: false);
753 HwasanMemTransferFnTy =
754 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy, Int8Ty}, isVarArg: false);
755 HwasanMemsetFnTy =
756 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy, Int8Ty}, isVarArg: false);
757 } else {
758 HwasanMemoryAccessCallbackSizedFnTy =
759 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy}, isVarArg: false);
760 HwasanMemoryAccessCallbackFnTy =
761 FunctionType::get(Result: VoidTy, Params: {IntptrTy}, isVarArg: false);
762 HwasanMemTransferFnTy =
763 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy}, isVarArg: false);
764 HwasanMemsetFnTy =
765 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy}, isVarArg: false);
766 }
767
768 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
769 const std::string TypeStr = AccessIsWrite ? "store" : "load";
770 const std::string EndingStr = Recover ? "_noabort" : "";
771
772 HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
773 Name: ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
774 T: HwasanMemoryAccessCallbackSizedFnTy);
775
776 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
777 AccessSizeIndex++) {
778 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
779 M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + TypeStr +
780 itostr(X: 1ULL << AccessSizeIndex) +
781 MatchAllStr + EndingStr,
782 T: HwasanMemoryAccessCallbackFnTy);
783 }
784 }
785
786 const std::string MemIntrinCallbackPrefix =
787 (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
788 ? std::string("")
789 : ClMemoryAccessCallbackPrefix;
790
791 HwasanMemmove = M.getOrInsertFunction(
792 Name: MemIntrinCallbackPrefix + "memmove" + MatchAllStr, T: HwasanMemTransferFnTy);
793 HwasanMemcpy = M.getOrInsertFunction(
794 Name: MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, T: HwasanMemTransferFnTy);
795 HwasanMemset = M.getOrInsertFunction(
796 Name: MemIntrinCallbackPrefix + "memset" + MatchAllStr, T: HwasanMemsetFnTy);
797
798 HwasanTagMemoryFunc = M.getOrInsertFunction(Name: "__hwasan_tag_memory", RetTy: VoidTy,
799 Args: PtrTy, Args: Int8Ty, Args: IntptrTy);
800 HwasanGenerateTagFunc =
801 M.getOrInsertFunction(Name: "__hwasan_generate_tag", RetTy: Int8Ty);
802
803 HwasanRecordFrameRecordFunc =
804 M.getOrInsertFunction(Name: "__hwasan_add_frame_record", RetTy: VoidTy, Args: Int64Ty);
805
806 ShadowGlobal =
807 M.getOrInsertGlobal(Name: "__hwasan_shadow", Ty: ArrayType::get(ElementType: Int8Ty, NumElements: 0));
808
809 HwasanHandleVfork =
810 M.getOrInsertFunction(Name: "__hwasan_handle_vfork", RetTy: VoidTy, Args: IntptrTy);
811}
812
813Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
814 // An empty inline asm with input reg == output reg.
815 // An opaque no-op cast, basically.
816 // This prevents code bloat as a result of rematerializing trivial definitions
817 // such as constants or global addresses at every load and store.
818 InlineAsm *Asm =
819 InlineAsm::get(Ty: FunctionType::get(Result: PtrTy, Params: {Val->getType()}, isVarArg: false),
820 AsmString: StringRef(""), Constraints: StringRef("=r,0"),
821 /*hasSideEffects=*/false);
822 return IRB.CreateCall(Callee: Asm, Args: {Val}, Name: ".hwasan.shadow");
823}
824
825Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
826 return getOpaqueNoopCast(IRB, Val: ShadowGlobal);
827}
828
829Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
830 if (Mapping.isFixed()) {
831 return getOpaqueNoopCast(
832 IRB, Val: ConstantExpr::getIntToPtr(
833 C: ConstantInt::get(Ty: IntptrTy, V: Mapping.offset()), Ty: PtrTy));
834 }
835
836 if (Mapping.isInIfunc())
837 return getDynamicShadowIfunc(IRB);
838
839 Value *GlobalDynamicAddress =
840 IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
841 Name: kHwasanShadowMemoryDynamicAddress, Ty: PtrTy);
842 return IRB.CreateLoad(Ty: PtrTy, Ptr: GlobalDynamicAddress);
843}
844
845bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst,
846 Value *Ptr) {
847 // Do not instrument accesses from different address spaces; we cannot deal
848 // with them.
849 Type *PtrTy = cast<PointerType>(Val: Ptr->getType()->getScalarType());
850 if (PtrTy->getPointerAddressSpace() != 0)
851 return true;
852
853 // Ignore swifterror addresses.
854 // swifterror memory addresses are mem2reg promoted by instruction
855 // selection. As such they cannot have regular uses like an instrumentation
856 // function and it makes no sense to track them as memory.
857 if (Ptr->isSwiftError())
858 return true;
859
860 if (findAllocaForValue(V: Ptr)) {
861 if (!InstrumentStack)
862 return true;
863 if (SSI && SSI->stackAccessIsSafe(I: *Inst))
864 return true;
865 }
866
867 if (isa<GlobalVariable>(Val: getUnderlyingObject(V: Ptr))) {
868 if (!InstrumentGlobals)
869 return true;
870 // TODO: Optimize inbound global accesses, like Asan `instrumentMop`.
871 }
872
873 return false;
874}
875
876bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE,
877 Instruction *Inst, Value *Ptr) {
878 bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr);
879 if (Ignored) {
880 ORE.emit(
881 RemarkBuilder: [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); });
882 } else {
883 ORE.emit(RemarkBuilder: [&]() {
884 return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst);
885 });
886 }
887 return Ignored;
888}
889
890void HWAddressSanitizer::getInterestingMemoryOperands(
891 OptimizationRemarkEmitter &ORE, Instruction *I,
892 const TargetLibraryInfo &TLI,
893 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
894 // Skip memory accesses inserted by another instrumentation.
895 if (I->hasMetadata(KindID: LLVMContext::MD_nosanitize))
896 return;
897
898 // Do not instrument the load fetching the dynamic shadow address.
899 if (ShadowBase == I)
900 return;
901
902 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
903 if (!ClInstrumentReads || ignoreAccess(ORE, Inst: I, Ptr: LI->getPointerOperand()))
904 return;
905 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
906 Args: LI->getType(), Args: LI->getAlign());
907 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
908 if (!ClInstrumentWrites || ignoreAccess(ORE, Inst: I, Ptr: SI->getPointerOperand()))
909 return;
910 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
911 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
912 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
913 if (!ClInstrumentAtomics || ignoreAccess(ORE, Inst: I, Ptr: RMW->getPointerOperand()))
914 return;
915 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
916 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
917 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
918 if (!ClInstrumentAtomics || ignoreAccess(ORE, Inst: I, Ptr: XCHG->getPointerOperand()))
919 return;
920 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
921 Args: XCHG->getCompareOperand()->getType(),
922 Args: std::nullopt);
923 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
924 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
925 if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
926 ignoreAccess(ORE, Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
927 continue;
928 Type *Ty = CI->getParamByValType(ArgNo);
929 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
930 }
931 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
932 }
933}
934
935static unsigned getPointerOperandIndex(Instruction *I) {
936 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I))
937 return LI->getPointerOperandIndex();
938 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I))
939 return SI->getPointerOperandIndex();
940 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I))
941 return RMW->getPointerOperandIndex();
942 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I))
943 return XCHG->getPointerOperandIndex();
944 report_fatal_error(reason: "Unexpected instruction");
945 return -1;
946}
947
948static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
949 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
950 assert(Res < kNumberOfAccessSizes);
951 return Res;
952}
953
954void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
955 if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
956 TargetTriple.isRISCV64())
957 return;
958
959 IRBuilder<> IRB(I);
960 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
961 Value *UntaggedPtr =
962 IRB.CreateIntToPtr(V: untagPointer(IRB, PtrLong: AddrLong), DestTy: Addr->getType());
963 I->setOperand(i: getPointerOperandIndex(I), Val: UntaggedPtr);
964}
965
966Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
967 // Mem >> Scale
968 Value *Shadow = IRB.CreateLShr(LHS: Mem, RHS: Mapping.scale());
969 if (Mapping.isFixed() && Mapping.offset() == 0)
970 return IRB.CreateIntToPtr(V: Shadow, DestTy: PtrTy);
971 // (Mem >> Scale) + Offset
972 return IRB.CreatePtrAdd(Ptr: ShadowBase, Offset: Shadow);
973}
974
975int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
976 unsigned AccessSizeIndex) {
977 return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
978 (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
979 (MatchAllTag.value_or(u: 0) << HWASanAccessInfo::MatchAllShift) |
980 (Recover << HWASanAccessInfo::RecoverShift) |
981 (IsWrite << HWASanAccessInfo::IsWriteShift) |
982 (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
983}
984
985HWAddressSanitizer::ShadowTagCheckInfo
986HWAddressSanitizer::insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
987 DomTreeUpdater &DTU, LoopInfo *LI) {
988 ShadowTagCheckInfo R;
989
990 IRBuilder<> IRB(InsertBefore);
991
992 R.PtrLong = IRB.CreatePointerCast(V: Ptr, DestTy: IntptrTy);
993 R.PtrTag =
994 IRB.CreateTrunc(V: IRB.CreateLShr(LHS: R.PtrLong, RHS: PointerTagShift), DestTy: Int8Ty);
995 R.AddrLong = untagPointer(IRB, PtrLong: R.PtrLong);
996 Value *Shadow = memToShadow(Mem: R.AddrLong, IRB);
997 R.MemTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: Shadow);
998 Value *TagMismatch = IRB.CreateICmpNE(LHS: R.PtrTag, RHS: R.MemTag);
999
1000 if (MatchAllTag.has_value()) {
1001 Value *TagNotIgnored = IRB.CreateICmpNE(
1002 LHS: R.PtrTag, RHS: ConstantInt::get(Ty: R.PtrTag->getType(), V: *MatchAllTag));
1003 TagMismatch = IRB.CreateAnd(LHS: TagMismatch, RHS: TagNotIgnored);
1004 }
1005
1006 R.TagMismatchTerm = SplitBlockAndInsertIfThen(
1007 Cond: TagMismatch, SplitBefore: InsertBefore, Unreachable: false,
1008 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
1009
1010 return R;
1011}
1012
1013void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
1014 unsigned AccessSizeIndex,
1015 Instruction *InsertBefore,
1016 DomTreeUpdater &DTU,
1017 LoopInfo *LI) {
1018 assert(!UsePageAliases);
1019 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
1020
1021 if (InlineFastPath)
1022 InsertBefore =
1023 insertShadowTagCheck(Ptr, InsertBefore, DTU, LI).TagMismatchTerm;
1024
1025 IRBuilder<> IRB(InsertBefore);
1026 bool UseFixedShadowIntrinsic = false;
1027 // The memaccess fixed shadow intrinsic is only supported on AArch64,
1028 // which allows a 16-bit immediate to be left-shifted by 32.
1029 // Since kShadowBaseAlignment == 32, and Linux by default will not
1030 // mmap above 48-bits, practically any valid shadow offset is
1031 // representable.
1032 // In particular, an offset of 4TB (1024 << 32) is representable, and
1033 // ought to be good enough for anybody.
1034 if (TargetTriple.isAArch64() && Mapping.isFixed()) {
1035 uint16_t OffsetShifted = Mapping.offset() >> 32;
1036 UseFixedShadowIntrinsic =
1037 static_cast<uint64_t>(OffsetShifted) << 32 == Mapping.offset();
1038 }
1039
1040 if (UseFixedShadowIntrinsic) {
1041 IRB.CreateIntrinsic(
1042 ID: UseShortGranules
1043 ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
1044 : Intrinsic::hwasan_check_memaccess_fixedshadow,
1045 Args: {Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo),
1046 ConstantInt::get(Ty: Int64Ty, V: Mapping.offset())});
1047 } else {
1048 IRB.CreateIntrinsic(
1049 ID: UseShortGranules ? Intrinsic::hwasan_check_memaccess_shortgranules
1050 : Intrinsic::hwasan_check_memaccess,
1051 Args: {ShadowBase, Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo)});
1052 }
1053}
1054
1055void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
1056 unsigned AccessSizeIndex,
1057 Instruction *InsertBefore,
1058 DomTreeUpdater &DTU,
1059 LoopInfo *LI) {
1060 assert(!UsePageAliases);
1061 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
1062
1063 ShadowTagCheckInfo TCI = insertShadowTagCheck(Ptr, InsertBefore, DTU, LI);
1064
1065 IRBuilder<> IRB(TCI.TagMismatchTerm);
1066 Value *OutOfShortGranuleTagRange =
1067 IRB.CreateICmpUGT(LHS: TCI.MemTag, RHS: ConstantInt::get(Ty: Int8Ty, V: 15));
1068 Instruction *CheckFailTerm = SplitBlockAndInsertIfThen(
1069 Cond: OutOfShortGranuleTagRange, SplitBefore: TCI.TagMismatchTerm, Unreachable: !Recover,
1070 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
1071
1072 IRB.SetInsertPoint(TCI.TagMismatchTerm);
1073 Value *PtrLowBits = IRB.CreateTrunc(V: IRB.CreateAnd(LHS: TCI.PtrLong, RHS: 15), DestTy: Int8Ty);
1074 PtrLowBits = IRB.CreateAdd(
1075 LHS: PtrLowBits, RHS: ConstantInt::get(Ty: Int8Ty, V: (1 << AccessSizeIndex) - 1));
1076 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(LHS: PtrLowBits, RHS: TCI.MemTag);
1077 SplitBlockAndInsertIfThen(Cond: PtrLowBitsOOB, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
1078 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
1079 LI, ThenBlock: CheckFailTerm->getParent());
1080
1081 IRB.SetInsertPoint(TCI.TagMismatchTerm);
1082 Value *InlineTagAddr = IRB.CreateOr(LHS: TCI.AddrLong, RHS: 15);
1083 InlineTagAddr = IRB.CreateIntToPtr(V: InlineTagAddr, DestTy: PtrTy);
1084 Value *InlineTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: InlineTagAddr);
1085 Value *InlineTagMismatch = IRB.CreateICmpNE(LHS: TCI.PtrTag, RHS: InlineTag);
1086 SplitBlockAndInsertIfThen(Cond: InlineTagMismatch, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
1087 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
1088 LI, ThenBlock: CheckFailTerm->getParent());
1089
1090 IRB.SetInsertPoint(CheckFailTerm);
1091 InlineAsm *Asm;
1092 switch (TargetTriple.getArch()) {
1093 case Triple::x86_64:
1094 // The signal handler will find the data address in rdi.
1095 Asm = InlineAsm::get(
1096 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1097 AsmString: "int3\nnopl " +
1098 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
1099 "(%rax)",
1100 Constraints: "{rdi}",
1101 /*hasSideEffects=*/true);
1102 break;
1103 case Triple::aarch64:
1104 case Triple::aarch64_be:
1105 // The signal handler will find the data address in x0.
1106 Asm = InlineAsm::get(
1107 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1108 AsmString: "brk #" + itostr(X: 0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
1109 Constraints: "{x0}",
1110 /*hasSideEffects=*/true);
1111 break;
1112 case Triple::riscv64:
1113 // The signal handler will find the data address in x10.
1114 Asm = InlineAsm::get(
1115 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1116 AsmString: "ebreak\naddiw x0, x11, " +
1117 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
1118 Constraints: "{x10}",
1119 /*hasSideEffects=*/true);
1120 break;
1121 default:
1122 report_fatal_error(reason: "unsupported architecture");
1123 }
1124 IRB.CreateCall(Callee: Asm, Args: TCI.PtrLong);
1125 if (Recover)
1126 cast<UncondBrInst>(Val: CheckFailTerm)
1127 ->setSuccessor(TCI.TagMismatchTerm->getParent());
1128}
1129
1130bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE,
1131 MemIntrinsic *MI) {
1132 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Val: MI)) {
1133 return (!ClInstrumentWrites || ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getDest())) &&
1134 (!ClInstrumentReads || ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getSource()));
1135 }
1136 if (isa<MemSetInst>(Val: MI))
1137 return !ClInstrumentWrites || ignoreAccess(ORE, Inst: MI, Ptr: MI->getDest());
1138 return false;
1139}
1140
1141void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
1142 IRBuilder<> IRB(MI);
1143 if (isa<MemTransferInst>(Val: MI)) {
1144 SmallVector<Value *, 4> Args{
1145 MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
1146 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1147
1148 if (UseMatchAllCallback)
1149 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1150 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? HwasanMemmove : HwasanMemcpy, Args);
1151 } else if (isa<MemSetInst>(Val: MI)) {
1152 SmallVector<Value *, 4> Args{
1153 MI->getOperand(i_nocapture: 0),
1154 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1155 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1156 if (UseMatchAllCallback)
1157 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1158 IRB.CreateCall(Callee: HwasanMemset, Args);
1159 }
1160 MI->eraseFromParent();
1161}
1162
1163bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O,
1164 DomTreeUpdater &DTU, LoopInfo *LI,
1165 const DataLayout &DL) {
1166 Value *Addr = O.getPtr();
1167
1168 LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
1169
1170 // If the pointer is statically known to be zero, the tag check will pass
1171 // since:
1172 // 1) it has a zero tag
1173 // 2) the shadow memory corresponding to address 0 is initialized to zero and
1174 // never updated.
1175 // We can therefore elide the tag check.
1176 llvm::KnownBits Known(DL.getPointerTypeSizeInBits(Addr->getType()));
1177 llvm::computeKnownBits(V: Addr, Known, DL);
1178 if (Known.isZero())
1179 return false;
1180
1181 if (O.MaybeMask)
1182 return false; // FIXME
1183
1184 IRBuilder<> IRB(O.getInsn());
1185 if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(Value: O.TypeStoreSize) &&
1186 (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1187 (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
1188 *O.Alignment >= O.TypeStoreSize / 8)) {
1189 size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize: O.TypeStoreSize);
1190 if (InstrumentWithCalls) {
1191 SmallVector<Value *, 2> Args{IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy)};
1192 if (UseMatchAllCallback)
1193 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1194 IRB.CreateCall(Callee: HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1195 Args);
1196 } else if (OutlinedChecks) {
1197 instrumentMemAccessOutline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1198 DTU, LI);
1199 } else {
1200 instrumentMemAccessInline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1201 DTU, LI);
1202 }
1203 } else {
1204 SmallVector<Value *, 3> Args{
1205 IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy),
1206 IRB.CreateUDiv(LHS: IRB.CreateTypeSize(Ty: IntptrTy, Size: O.TypeStoreSize),
1207 RHS: ConstantInt::get(Ty: IntptrTy, V: 8))};
1208 if (UseMatchAllCallback)
1209 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1210 IRB.CreateCall(Callee: HwasanMemoryAccessCallbackSized[O.IsWrite], Args);
1211 }
1212 untagPointerOperand(I: O.getInsn(), Addr);
1213
1214 return true;
1215}
1216
1217void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1218 size_t Size) {
1219 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1220 if (!UseShortGranules)
1221 Size = AlignedSize;
1222
1223 Tag = IRB.CreateTrunc(V: Tag, DestTy: Int8Ty);
1224 if (InstrumentWithCalls) {
1225 IRB.CreateCall(Callee: HwasanTagMemoryFunc,
1226 Args: {IRB.CreatePointerCast(V: AI, DestTy: PtrTy), Tag,
1227 ConstantInt::get(Ty: IntptrTy, V: AlignedSize)});
1228 } else {
1229 size_t ShadowSize = Size >> Mapping.scale();
1230 Value *AddrLong = untagPointer(IRB, PtrLong: IRB.CreatePointerCast(V: AI, DestTy: IntptrTy));
1231 Value *ShadowPtr = memToShadow(Mem: AddrLong, IRB);
1232 // If this memset is not inlined, it will be intercepted in the hwasan
1233 // runtime library. That's OK, because the interceptor skips the checks if
1234 // the address is in the shadow region.
1235 // FIXME: the interceptor is not as fast as real memset. Consider lowering
1236 // llvm.memset right here into either a sequence of stores, or a call to
1237 // hwasan_tag_memory.
1238 if (ShadowSize)
1239 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Tag, Size: ShadowSize, Align: Align(1));
1240 if (Size != AlignedSize) {
1241 const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1242 IRB.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: SizeRemainder),
1243 Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: ShadowPtr, Idx0: ShadowSize));
1244 IRB.CreateStore(
1245 Val: Tag, Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: IRB.CreatePointerCast(V: AI, DestTy: PtrTy),
1246 Idx0: AlignedSize - 1));
1247 }
1248 }
1249}
1250
1251unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1252 if (TargetTriple.getArch() == Triple::x86_64)
1253 return AllocaNo & TagMaskByte;
1254
1255 // A list of 8-bit numbers that have at most one run of non-zero bits.
1256 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1257 // masks.
1258 // The list does not include the value 255, which is used for UAR.
1259 //
1260 // Because we are more likely to use earlier elements of this list than later
1261 // ones, it is sorted in increasing order of probability of collision with a
1262 // mask allocated (temporally) nearby. The program that generated this list
1263 // can be found at:
1264 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1265 static const unsigned FastMasks[] = {
1266 0, 128, 64, 192, 32, 96, 224, 112, 240, 48, 16, 120,
1267 248, 56, 24, 8, 124, 252, 60, 28, 12, 4, 126, 254,
1268 62, 30, 14, 6, 2, 127, 63, 31, 15, 7, 3, 1};
1269 return FastMasks[AllocaNo % std::size(FastMasks)];
1270}
1271
1272Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1273 if (TagMaskByte == 0xFF)
1274 return OldTag; // No need to clear the tag byte.
1275 return IRB.CreateAnd(LHS: OldTag,
1276 RHS: ConstantInt::get(Ty: OldTag->getType(), V: TagMaskByte));
1277}
1278
1279Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1280 return IRB.CreateZExt(V: IRB.CreateCall(Callee: HwasanGenerateTagFunc), DestTy: IntptrTy);
1281}
1282
1283Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1284 if (ClGenerateTagsWithCalls)
1285 return nullptr;
1286 if (StackBaseTag)
1287 return StackBaseTag;
1288 // Extract some entropy from the stack pointer for the tags.
1289 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1290 // between functions).
1291 Value *FramePointerLong = getCachedFP(IRB);
1292 Value *StackTag =
1293 applyTagMask(IRB, OldTag: IRB.CreateXor(LHS: FramePointerLong,
1294 RHS: IRB.CreateLShr(LHS: FramePointerLong, RHS: 20)));
1295 StackTag->setName("hwasan.stack.base.tag");
1296 return StackTag;
1297}
1298
1299Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1300 unsigned AllocaNo) {
1301 if (ClGenerateTagsWithCalls)
1302 return getNextTagWithCall(IRB);
1303 return IRB.CreateXor(
1304 LHS: StackTag, RHS: ConstantInt::get(Ty: StackTag->getType(), V: retagMask(AllocaNo)));
1305}
1306
1307Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
1308 Value *FramePointerLong = getCachedFP(IRB);
1309 Value *UARTag =
1310 applyTagMask(IRB, OldTag: IRB.CreateLShr(LHS: FramePointerLong, RHS: PointerTagShift));
1311
1312 UARTag->setName("hwasan.uar.tag");
1313 return UARTag;
1314}
1315
1316// Add a tag to an address.
1317Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1318 Value *PtrLong, Value *Tag) {
1319 assert(!UsePageAliases);
1320 Value *TaggedPtrLong;
1321 if (CompileKernel) {
1322 // Kernel addresses have 0xFF in the most significant byte.
1323 Value *ShiftedTag =
1324 IRB.CreateOr(LHS: IRB.CreateShl(LHS: Tag, RHS: PointerTagShift),
1325 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << PointerTagShift) - 1));
1326 TaggedPtrLong = IRB.CreateAnd(LHS: PtrLong, RHS: ShiftedTag);
1327 } else {
1328 // Userspace can simply do OR (tag << PointerTagShift);
1329 Value *ShiftedTag = IRB.CreateShl(LHS: Tag, RHS: PointerTagShift);
1330 TaggedPtrLong = IRB.CreateOr(LHS: PtrLong, RHS: ShiftedTag);
1331 }
1332 return IRB.CreateIntToPtr(V: TaggedPtrLong, DestTy: Ty);
1333}
1334
1335// Remove tag from an address.
1336Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1337 assert(!UsePageAliases);
1338 Value *UntaggedPtrLong;
1339 if (CompileKernel) {
1340 // Kernel addresses have 0xFF in the most significant byte.
1341 UntaggedPtrLong =
1342 IRB.CreateOr(LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1343 V: TagMaskByte << PointerTagShift));
1344 } else {
1345 // Userspace addresses have 0x00.
1346 UntaggedPtrLong = IRB.CreateAnd(
1347 LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1348 V: ~(TagMaskByte << PointerTagShift)));
1349 }
1350 return UntaggedPtrLong;
1351}
1352
1353Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) {
1354 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1355 // in Bionic's libc/platform/bionic/tls_defines.h.
1356 constexpr int SanitizerSlot = 6;
1357 if (TargetTriple.isAArch64() && TargetTriple.isAndroid())
1358 return memtag::getAndroidSlotPtr(IRB, Slot: SanitizerSlot);
1359 return ThreadPtrGlobal;
1360}
1361
1362Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) {
1363 if (!CachedFP)
1364 CachedFP = memtag::getFP(IRB);
1365 return CachedFP;
1366}
1367
1368Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1369 // Prepare ring buffer data.
1370 Value *PC = memtag::getPC(TargetTriple, IRB);
1371 Value *FP = getCachedFP(IRB);
1372
1373 // Mix FP and PC.
1374 // Assumptions:
1375 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
1376 // FP is 0xfffffffffffFFFF0 (4 lower bits are zero)
1377 // We only really need ~20 lower non-zero bits (FFFF), so we mix like this:
1378 // 0xFFFFPPPPPPPPPPPP
1379 //
1380 // FP works because in AArch64FrameLowering::getFrameIndexReference, we
1381 // prefer FP-relative offsets for functions compiled with HWASan.
1382 FP = IRB.CreateShl(LHS: FP, RHS: 44);
1383 return IRB.CreateOr(LHS: PC, RHS: FP);
1384}
1385
1386void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1387 if (!Mapping.isInTls())
1388 ShadowBase = getShadowNonTls(IRB);
1389 else if (!WithFrameRecord && TargetTriple.isAndroid())
1390 ShadowBase = getDynamicShadowIfunc(IRB);
1391
1392 if (!WithFrameRecord && ShadowBase)
1393 return;
1394
1395 Value *SlotPtr = nullptr;
1396 Value *ThreadLong = nullptr;
1397 Value *ThreadLongMaybeUntagged = nullptr;
1398
1399 auto getThreadLongMaybeUntagged = [&]() {
1400 if (!SlotPtr)
1401 SlotPtr = getHwasanThreadSlotPtr(IRB);
1402 if (!ThreadLong)
1403 ThreadLong = IRB.CreateLoad(Ty: IntptrTy, Ptr: SlotPtr);
1404 // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1405 // TBI.
1406 return TargetTriple.isAArch64() ? ThreadLong
1407 : untagPointer(IRB, PtrLong: ThreadLong);
1408 };
1409
1410 if (WithFrameRecord) {
1411 switch (ClRecordStackHistory) {
1412 case libcall: {
1413 // Emit a runtime call into hwasan rather than emitting instructions for
1414 // recording stack history.
1415 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1416 IRB.CreateCall(Callee: HwasanRecordFrameRecordFunc, Args: {FrameRecordInfo});
1417 break;
1418 }
1419 case instr: {
1420 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1421
1422 StackBaseTag = IRB.CreateAShr(LHS: ThreadLong, RHS: 3);
1423
1424 // Store data to ring buffer.
1425 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1426 Value *RecordPtr =
1427 IRB.CreateIntToPtr(V: ThreadLongMaybeUntagged, DestTy: IRB.getPtrTy(AddrSpace: 0));
1428 IRB.CreateStore(Val: FrameRecordInfo, Ptr: RecordPtr);
1429
1430 IRB.CreateStore(Val: memtag::incrementThreadLong(IRB, ThreadLong, Inc: 8), Ptr: SlotPtr);
1431 break;
1432 }
1433 case none: {
1434 llvm_unreachable(
1435 "A stack history recording mode should've been selected.");
1436 }
1437 }
1438 }
1439
1440 if (!ShadowBase) {
1441 if (!ThreadLongMaybeUntagged)
1442 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1443
1444 // Get shadow base address by aligning RecordPtr up.
1445 // Note: this is not correct if the pointer is already aligned.
1446 // Runtime library will make sure this never happens.
1447 ShadowBase = IRB.CreateAdd(
1448 LHS: IRB.CreateOr(
1449 LHS: ThreadLongMaybeUntagged,
1450 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << kShadowBaseAlignment) - 1)),
1451 RHS: ConstantInt::get(Ty: IntptrTy, V: 1), Name: "hwasan.shadow");
1452 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: PtrTy);
1453 }
1454}
1455
1456void HWAddressSanitizer::instrumentLandingPads(
1457 SmallVectorImpl<Instruction *> &LandingPadVec) {
1458 for (auto *LP : LandingPadVec) {
1459 IRBuilder<> IRB(LP->getNextNode());
1460 IRB.CreateCall(
1461 Callee: HwasanHandleVfork,
1462 Args: {memtag::readRegister(
1463 IRB, Name: (TargetTriple.getArch() == Triple::x86_64) ? "rsp" : "sp")});
1464 }
1465}
1466
1467void HWAddressSanitizer::instrumentStack(OptimizationRemarkEmitter &ORE,
1468 memtag::StackInfo &SInfo,
1469 Value *StackTag, Value *UARTag,
1470 const DominatorTree &DT,
1471 const PostDominatorTree &PDT,
1472 const LoopInfo &LI) {
1473 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1474 // alloca addresses using that. Unfortunately, offsets are not known yet
1475 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1476 // temp, shift-OR it into each alloca address and xor with the retag mask.
1477 // This generates one extra instruction per alloca use.
1478 unsigned int I = 0;
1479
1480 for (auto &KV : SInfo.AllocasToInstrument) {
1481 auto N = I++;
1482 auto *AI = KV.first;
1483 memtag::AllocaInfo &Info = KV.second;
1484 IRBuilder<> IRB(AI->getNextNode());
1485
1486 // Replace uses of the alloca with tagged address.
1487 Value *Tag = getAllocaTag(IRB, StackTag, AllocaNo: N);
1488 Value *AILong = IRB.CreatePointerCast(V: AI, DestTy: IntptrTy);
1489 Value *AINoTagLong = untagPointer(IRB, PtrLong: AILong);
1490 Value *Replacement = tagPointer(IRB, Ty: AI->getType(), PtrLong: AINoTagLong, Tag);
1491 std::string Name =
1492 AI->hasName() ? AI->getName().str() : "alloca." + itostr(X: N);
1493 Replacement->setName(Name + ".hwasan");
1494
1495 size_t Size = memtag::getAllocaSizeInBytes(AI: *AI);
1496 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1497
1498 AI->replaceUsesWithIf(New: Replacement, ShouldReplace: [AILong](const Use &U) {
1499 auto *User = U.getUser();
1500 return User != AILong && !isa<LifetimeIntrinsic>(Val: User);
1501 });
1502
1503 memtag::annotateDebugRecords(Info, Tag: retagMask(AllocaNo: N));
1504
1505 auto TagStarts = [&]() {
1506 for (IntrinsicInst *Start : Info.LifetimeStart) {
1507 IRB.SetInsertPoint(Start->getNextNode());
1508 tagAlloca(IRB, AI, Tag, Size);
1509 }
1510 };
1511 auto TagEnd = [&](Instruction *Node) {
1512 IRB.SetInsertPoint(Node);
1513 // When untagging, use the `AlignedSize` because we need to set the tags
1514 // for the entire alloca to original. If we used `Size` here, we would
1515 // keep the last granule tagged, and store zero in the last byte of the
1516 // last granule, due to how short granules are implemented.
1517 tagAlloca(IRB, AI, Tag: UARTag, Size: AlignedSize);
1518 };
1519 auto EraseLifetimes = [&]() {
1520 for (auto &II : Info.LifetimeStart)
1521 II->eraseFromParent();
1522 for (auto &II : Info.LifetimeEnd)
1523 II->eraseFromParent();
1524 };
1525 // Calls to functions that may return twice (e.g. setjmp) confuse the
1526 // postdominator analysis, and will leave us to keep memory tagged after
1527 // function return. Work around this by always untagging at every return
1528 // statement if return_twice functions are called.
1529 if (DetectUseAfterScope && !SInfo.CallsReturnTwice &&
1530 memtag::isSupportedLifetime(AInfo: Info, DT: &DT, LI: &LI)) {
1531 TagStarts();
1532 memtag::forAllReachableExits(DT, PDT, LI, AInfo: Info, RetVec: SInfo.RetVec, Callback: TagEnd);
1533 ORE.emit(RemarkBuilder: [&]() {
1534 return OptimizationRemark(DEBUG_TYPE, "supportedLifetime", AI);
1535 });
1536 } else if (DetectUseAfterScope && ClStrictUseAfterScope) {
1537 // SInfo.CallsReturnTwice || !isStandardLifetime
1538 ORE.emit(RemarkBuilder: [&]() {
1539 return OptimizationRemarkMissed(DEBUG_TYPE, "supportedLifetime", AI);
1540 });
1541
1542 tagAlloca(IRB, AI, Tag, Size);
1543 TagStarts();
1544 for_each(Range&: Info.LifetimeEnd, F: TagEnd);
1545 for_each(Range&: SInfo.RetVec, F: TagEnd);
1546 EraseLifetimes();
1547 } else {
1548 tagAlloca(IRB, AI, Tag, Size);
1549 for_each(Range&: SInfo.RetVec, F: TagEnd);
1550 EraseLifetimes();
1551 }
1552 memtag::alignAndPadAlloca(Info, Align: Mapping.getObjectAlignment());
1553 }
1554}
1555
1556static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE,
1557 bool Skip) {
1558 if (Skip) {
1559 ORE.emit(RemarkBuilder: [&]() {
1560 return OptimizationRemark(DEBUG_TYPE, "Skip", &F)
1561 << "Skipped: F=" << ore::NV("Function", &F);
1562 });
1563 } else {
1564 ORE.emit(RemarkBuilder: [&]() {
1565 return OptimizationRemarkMissed(DEBUG_TYPE, "Sanitize", &F)
1566 << "Sanitized: F=" << ore::NV("Function", &F);
1567 });
1568 }
1569}
1570
1571bool HWAddressSanitizer::selectiveInstrumentationShouldSkip(
1572 Function &F, FunctionAnalysisManager &FAM) const {
1573 auto SkipHot = [&]() {
1574 if (!ClHotPercentileCutoff.getNumOccurrences())
1575 return false;
1576 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1577 ProfileSummaryInfo *PSI =
1578 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
1579 if (!PSI || !PSI->hasProfileSummary()) {
1580 ++NumNoProfileSummaryFuncs;
1581 return false;
1582 }
1583 return PSI->isFunctionHotInCallGraphNthPercentile(
1584 PercentileCutoff: ClHotPercentileCutoff, F: &F, BFI&: FAM.getResult<BlockFrequencyAnalysis>(IR&: F));
1585 };
1586
1587 auto SkipRandom = [&]() {
1588 if (!ClRandomKeepRate.getNumOccurrences())
1589 return false;
1590 std::bernoulli_distribution D(ClRandomKeepRate);
1591 return !D(*Rng);
1592 };
1593
1594 bool Skip = SkipRandom() || SkipHot();
1595 emitRemark(F, ORE&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F), Skip);
1596 return Skip;
1597}
1598
1599void HWAddressSanitizer::sanitizeFunction(Function &F,
1600 FunctionAnalysisManager &FAM) {
1601 if (&F == HwasanCtorFunction)
1602 return;
1603
1604 // Do not apply any instrumentation for naked functions.
1605 if (F.hasFnAttribute(Kind: Attribute::Naked))
1606 return;
1607
1608 if (!F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1609 return;
1610
1611 if (F.empty())
1612 return;
1613
1614 if (F.isPresplitCoroutine())
1615 return;
1616
1617 NumTotalFuncs++;
1618
1619 OptimizationRemarkEmitter &ORE =
1620 FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
1621
1622 if (selectiveInstrumentationShouldSkip(F, FAM))
1623 return;
1624
1625 NumInstrumentedFuncs++;
1626
1627 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1628
1629 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1630 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1631 SmallVector<Instruction *, 8> LandingPadVec;
1632 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1633
1634 memtag::StackInfoBuilder SIB(SSI, DEBUG_TYPE);
1635 for (auto &Inst : instructions(F)) {
1636 if (InstrumentStack) {
1637 SIB.visit(ORE, Inst);
1638 }
1639
1640 if (InstrumentLandingPads && isa<LandingPadInst>(Val: Inst))
1641 LandingPadVec.push_back(Elt: &Inst);
1642
1643 getInterestingMemoryOperands(ORE, I: &Inst, TLI, Interesting&: OperandsToInstrument);
1644
1645 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst))
1646 if (!ignoreMemIntrinsic(ORE, MI))
1647 IntrinToInstrument.push_back(Elt: MI);
1648 }
1649
1650 memtag::StackInfo &SInfo = SIB.get();
1651
1652 initializeCallbacks(M&: *F.getParent());
1653
1654 if (!LandingPadVec.empty())
1655 instrumentLandingPads(LandingPadVec);
1656
1657 if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1658 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1659 // __hwasan_personality_thunk is a no-op for functions without an
1660 // instrumented stack, so we can drop it.
1661 F.setPersonalityFn(nullptr);
1662 }
1663
1664 if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1665 IntrinToInstrument.empty())
1666 return;
1667
1668 assert(!ShadowBase);
1669
1670 BasicBlock::iterator InsertPt = F.getEntryBlock().begin();
1671 IRBuilder<> EntryIRB(&F.getEntryBlock(), InsertPt);
1672 emitPrologue(IRB&: EntryIRB,
1673 /*WithFrameRecord*/ ClRecordStackHistory != none &&
1674 Mapping.withFrameRecord() &&
1675 !SInfo.AllocasToInstrument.empty());
1676
1677 if (!SInfo.AllocasToInstrument.empty()) {
1678 const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
1679 const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
1680 const LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: F);
1681 Value *StackTag = getStackBaseTag(IRB&: EntryIRB);
1682 Value *UARTag = getUARTag(IRB&: EntryIRB);
1683 instrumentStack(ORE, SInfo, StackTag, UARTag, DT, PDT, LI);
1684 }
1685
1686 // If we split the entry block, move any allocas that were originally in the
1687 // entry block back into the entry block so that they aren't treated as
1688 // dynamic allocas.
1689 if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1690 InsertPt = F.getEntryBlock().begin();
1691 for (Instruction &I :
1692 llvm::make_early_inc_range(Range&: *EntryIRB.GetInsertBlock())) {
1693 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
1694 if (isa<ConstantInt>(Val: AI->getArraySize()))
1695 I.moveBefore(BB&: F.getEntryBlock(), I: InsertPt);
1696 }
1697 }
1698
1699 DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
1700 PostDominatorTree *PDT = FAM.getCachedResult<PostDominatorTreeAnalysis>(IR&: F);
1701 LoopInfo *LI = FAM.getCachedResult<LoopAnalysis>(IR&: F);
1702 DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
1703 const DataLayout &DL = F.getDataLayout();
1704 for (auto &Operand : OperandsToInstrument)
1705 instrumentMemAccess(O&: Operand, DTU, LI, DL);
1706 DTU.flush();
1707
1708 if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1709 for (auto *Inst : IntrinToInstrument)
1710 instrumentMemIntrinsic(MI: Inst);
1711 }
1712
1713 ShadowBase = nullptr;
1714 StackBaseTag = nullptr;
1715 CachedFP = nullptr;
1716}
1717
1718void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1719 assert(!UsePageAliases);
1720 Constant *Initializer = GV->getInitializer();
1721 uint64_t SizeInBytes =
1722 M.getDataLayout().getTypeAllocSize(Ty: Initializer->getType());
1723 uint64_t NewSize = alignTo(Size: SizeInBytes, A: Mapping.getObjectAlignment());
1724 if (SizeInBytes != NewSize) {
1725 // Pad the initializer out to the next multiple of 16 bytes and add the
1726 // required short granule tag.
1727 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1728 Init.back() = Tag;
1729 Constant *Padding = ConstantDataArray::get(Context&: *C, Elts&: Init);
1730 Initializer = ConstantStruct::getAnon(V: {Initializer, Padding});
1731 }
1732
1733 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1734 GlobalValue::ExternalLinkage, Initializer,
1735 GV->getName() + ".hwasan");
1736 NewGV->copyAttributesFrom(Src: GV);
1737 NewGV->setLinkage(GlobalValue::PrivateLinkage);
1738 NewGV->copyMetadata(Src: GV, Offset: 0);
1739 NewGV->setAlignment(
1740 std::max(a: GV->getAlign().valueOrOne(), b: Mapping.getObjectAlignment()));
1741
1742 // It is invalid to ICF two globals that have different tags. In the case
1743 // where the size of the global is a multiple of the tag granularity the
1744 // contents of the globals may be the same but the tags (i.e. symbol values)
1745 // may be different, and the symbols are not considered during ICF. In the
1746 // case where the size is not a multiple of the granularity, the short granule
1747 // tags would discriminate two globals with different tags, but there would
1748 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1749 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1750 // granule tag in the last byte.
1751 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1752
1753 // Descriptor format (assuming little-endian):
1754 // bytes 0-3: relative address of global
1755 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1756 // it isn't, we create multiple descriptors)
1757 // byte 7: tag
1758 auto *DescriptorTy = StructType::get(elt1: Int32Ty, elts: Int32Ty);
1759 const uint64_t MaxDescriptorSize = 0xfffff0;
1760 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1761 DescriptorPos += MaxDescriptorSize) {
1762 auto *Descriptor =
1763 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1764 nullptr, GV->getName() + ".hwasan.descriptor");
1765 auto *GVRelPtr = ConstantExpr::getTrunc(
1766 C: ConstantExpr::getAdd(
1767 C1: ConstantExpr::getSub(
1768 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1769 C2: ConstantExpr::getPtrToInt(C: Descriptor, Ty: Int64Ty)),
1770 C2: ConstantInt::get(Ty: Int64Ty, V: DescriptorPos)),
1771 Ty: Int32Ty);
1772 uint32_t Size = std::min(a: SizeInBytes - DescriptorPos, b: MaxDescriptorSize);
1773 auto *SizeAndTag = ConstantInt::get(Ty: Int32Ty, V: Size | (uint32_t(Tag) << 24));
1774 Descriptor->setComdat(NewGV->getComdat());
1775 Descriptor->setInitializer(ConstantStruct::getAnon(V: {GVRelPtr, SizeAndTag}));
1776 Descriptor->setSection("hwasan_globals");
1777 Descriptor->setMetadata(KindID: LLVMContext::MD_associated,
1778 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: NewGV)));
1779 appendToCompilerUsed(M, Values: Descriptor);
1780 }
1781
1782 Constant *Aliasee = ConstantExpr::getIntToPtr(
1783 C: ConstantExpr::getAdd(
1784 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1785 C2: ConstantInt::get(Ty: Int64Ty, V: uint64_t(Tag) << PointerTagShift)),
1786 Ty: GV->getType());
1787 auto *Alias = GlobalAlias::create(Ty: GV->getValueType(), AddressSpace: GV->getAddressSpace(),
1788 Linkage: GV->getLinkage(), Name: "", Aliasee, Parent: &M);
1789 Alias->setVisibility(GV->getVisibility());
1790 Alias->takeName(V: GV);
1791 GV->replaceAllUsesWith(V: Alias);
1792 GV->eraseFromParent();
1793}
1794
1795void HWAddressSanitizer::instrumentGlobals() {
1796 std::vector<GlobalVariable *> Globals;
1797 for (GlobalVariable &GV : M.globals()) {
1798 if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1799 continue;
1800
1801 if (GV.isDeclarationForLinker() || GV.getName().starts_with(Prefix: "llvm.") ||
1802 GV.isThreadLocal())
1803 continue;
1804
1805 // Common symbols can't have aliases point to them, so they can't be tagged.
1806 if (GV.hasCommonLinkage())
1807 continue;
1808
1809 if (ClAllGlobals) {
1810 // Avoid instrumenting intrinsic global variables.
1811 if (GV.getSection() == "llvm.metadata")
1812 continue;
1813 } else {
1814 // Globals with custom sections may be used in __start_/__stop_
1815 // enumeration, which would be broken both by adding tags and potentially
1816 // by the extra padding/alignment that we insert.
1817 if (GV.hasSection())
1818 continue;
1819 }
1820
1821 Globals.push_back(x: &GV);
1822 }
1823
1824 MD5 Hasher;
1825 Hasher.update(Str: M.getSourceFileName());
1826 MD5::MD5Result Hash;
1827 Hasher.final(Result&: Hash);
1828 uint8_t Tag = Hash[0];
1829
1830 assert(TagMaskByte >= 16);
1831
1832 for (GlobalVariable *GV : Globals) {
1833 // Don't allow globals to be tagged with something that looks like a
1834 // short-granule tag, otherwise we lose inter-granule overflow detection, as
1835 // the fast path shadow-vs-address check succeeds.
1836 if (Tag < 16 || Tag > TagMaskByte)
1837 Tag = 16;
1838 instrumentGlobal(GV, Tag: Tag++);
1839 }
1840}
1841
1842void HWAddressSanitizer::instrumentPersonalityFunctions() {
1843 // We need to untag stack frames as we unwind past them. That is the job of
1844 // the personality function wrapper, which either wraps an existing
1845 // personality function or acts as a personality function on its own. Each
1846 // function that has a personality function or that can be unwound past has
1847 // its personality function changed to a thunk that calls the personality
1848 // function wrapper in the runtime.
1849 MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1850 for (Function &F : M) {
1851 if (F.isDeclaration() || !F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1852 continue;
1853
1854 if (F.hasPersonalityFn()) {
1855 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(x: &F);
1856 } else if (!F.hasFnAttribute(Kind: Attribute::NoUnwind)) {
1857 PersonalityFns[nullptr].push_back(x: &F);
1858 }
1859 }
1860
1861 if (PersonalityFns.empty())
1862 return;
1863
1864 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1865 Name: "__hwasan_personality_wrapper", RetTy: Int32Ty, Args: Int32Ty, Args: Int32Ty, Args: Int64Ty, Args: PtrTy,
1866 Args: PtrTy, Args: PtrTy, Args: PtrTy, Args: PtrTy);
1867 FunctionCallee UnwindGetGR = M.getOrInsertFunction(Name: "_Unwind_GetGR", RetTy: VoidTy);
1868 FunctionCallee UnwindGetCFA = M.getOrInsertFunction(Name: "_Unwind_GetCFA", RetTy: VoidTy);
1869
1870 for (auto &P : PersonalityFns) {
1871 std::string ThunkName = kHwasanPersonalityThunkName;
1872 if (P.first)
1873 ThunkName += ("." + P.first->getName()).str();
1874 FunctionType *ThunkFnTy = FunctionType::get(
1875 Result: Int32Ty, Params: {Int32Ty, Int32Ty, Int64Ty, PtrTy, PtrTy}, isVarArg: false);
1876 bool IsLocal = P.first && (!isa<GlobalValue>(Val: P.first) ||
1877 cast<GlobalValue>(Val: P.first)->hasLocalLinkage());
1878 auto *ThunkFn = Function::Create(Ty: ThunkFnTy,
1879 Linkage: IsLocal ? GlobalValue::InternalLinkage
1880 : GlobalValue::LinkOnceODRLinkage,
1881 N: ThunkName, M: &M);
1882 // TODO: think about other attributes as well.
1883 if (any_of(Range&: P.second, P: [](const Function *F) {
1884 return F->hasFnAttribute(Kind: "branch-target-enforcement");
1885 })) {
1886 ThunkFn->addFnAttr(Kind: "branch-target-enforcement");
1887 }
1888 if (!IsLocal) {
1889 ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1890 ThunkFn->setComdat(M.getOrInsertComdat(Name: ThunkName));
1891 }
1892
1893 auto *BB = BasicBlock::Create(Context&: *C, Name: "entry", Parent: ThunkFn);
1894 IRBuilder<> IRB(BB);
1895 CallInst *WrapperCall = IRB.CreateCall(
1896 Callee: HwasanPersonalityWrapper,
1897 Args: {ThunkFn->getArg(i: 0), ThunkFn->getArg(i: 1), ThunkFn->getArg(i: 2),
1898 ThunkFn->getArg(i: 3), ThunkFn->getArg(i: 4),
1899 P.first ? P.first : Constant::getNullValue(Ty: PtrTy),
1900 UnwindGetGR.getCallee(), UnwindGetCFA.getCallee()});
1901 WrapperCall->setTailCall();
1902 IRB.CreateRet(V: WrapperCall);
1903
1904 for (Function *F : P.second)
1905 F->setPersonalityFn(ThunkFn);
1906 }
1907}
1908
1909void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1910 bool InstrumentWithCalls,
1911 bool CompileKernel) {
1912 // Start with defaults.
1913 Scale = kDefaultShadowScale;
1914 Kind = OffsetKind::kTls;
1915 WithFrameRecord = true;
1916
1917 // Tune for the target.
1918 if (TargetTriple.isOSFuchsia()) {
1919 // Fuchsia is always PIE, which means that the beginning of the address
1920 // space is always available.
1921 Kind = OffsetKind::kGlobal;
1922 } else if (CompileKernel || InstrumentWithCalls) {
1923 SetFixed(0);
1924 WithFrameRecord = false;
1925 }
1926
1927 WithFrameRecord = optOr(Opt&: ClFrameRecords, Other: WithFrameRecord);
1928
1929 // Apply the last of ClMappingOffset and ClMappingOffsetDynamic.
1930 Kind = optOr(Opt&: ClMappingOffsetDynamic, Other: Kind);
1931 if (ClMappingOffset.getNumOccurrences() > 0 &&
1932 !(ClMappingOffsetDynamic.getNumOccurrences() > 0 &&
1933 ClMappingOffsetDynamic.getPosition() > ClMappingOffset.getPosition())) {
1934 SetFixed(ClMappingOffset);
1935 }
1936}
1937