1//===- HWAddressSanitizer.cpp - memory access error detector --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of HWAddressSanitizer, an address basic correctness
11/// checker based on tagged addressing.
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15#include "llvm/ADT/MapVector.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/DomTreeUpdater.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ProfileSummaryInfo.h"
27#include "llvm/Analysis/StackSafetyAnalysis.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/BinaryFormat/Dwarf.h"
31#include "llvm/BinaryFormat/ELF.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/Dominators.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/InstIterator.h"
43#include "llvm/IR/Instruction.h"
44#include "llvm/IR/Instructions.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/LLVMContext.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/IR/Module.h"
50#include "llvm/IR/Type.h"
51#include "llvm/IR/Value.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Debug.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/MD5.h"
57#include "llvm/Support/RandomNumberGenerator.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/TargetParser/Triple.h"
60#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
61#include "llvm/Transforms/Utils/BasicBlockUtils.h"
62#include "llvm/Transforms/Utils/Instrumentation.h"
63#include "llvm/Transforms/Utils/Local.h"
64#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
65#include "llvm/Transforms/Utils/ModuleUtils.h"
66#include "llvm/Transforms/Utils/PromoteMemToReg.h"
67#include <optional>
68#include <random>
69
70using namespace llvm;
71
72#define DEBUG_TYPE "hwasan"
73
74const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
75const char kHwasanNoteName[] = "hwasan.note";
76const char kHwasanInitName[] = "__hwasan_init";
77const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
78
79const char kHwasanShadowMemoryDynamicAddress[] =
80 "__hwasan_shadow_memory_dynamic_address";
81
82// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
83static const size_t kNumberOfAccessSizes = 5;
84
85static const size_t kDefaultShadowScale = 4;
86
87static const unsigned kShadowBaseAlignment = 32;
88
89namespace {
90enum class OffsetKind {
91 kFixed = 0,
92 kGlobal,
93 kIfunc,
94 kTls,
95};
96}
97
98static cl::opt<std::string>
99 ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
100 cl::desc("Prefix for memory access callbacks"),
101 cl::Hidden, cl::init(Val: "__hwasan_"));
102
103static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
104 "hwasan-kernel-mem-intrinsic-prefix",
105 cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
106 cl::init(Val: false));
107
108static cl::opt<bool> ClInstrumentWithCalls(
109 "hwasan-instrument-with-calls",
110 cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
111 cl::init(Val: false));
112
113static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
114 cl::desc("instrument read instructions"),
115 cl::Hidden, cl::init(Val: true));
116
117static cl::opt<bool>
118 ClInstrumentWrites("hwasan-instrument-writes",
119 cl::desc("instrument write instructions"), cl::Hidden,
120 cl::init(Val: true));
121
122static cl::opt<bool> ClInstrumentAtomics(
123 "hwasan-instrument-atomics",
124 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
125 cl::init(Val: true));
126
127static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
128 cl::desc("instrument byval arguments"),
129 cl::Hidden, cl::init(Val: true));
130
131static cl::opt<bool>
132 ClRecover("hwasan-recover",
133 cl::desc("Enable recovery mode (continue-after-error)."),
134 cl::Hidden, cl::init(Val: false));
135
136static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
137 cl::desc("instrument stack (allocas)"),
138 cl::Hidden, cl::init(Val: true));
139
140static cl::opt<bool>
141 ClUseStackSafety("hwasan-use-stack-safety", cl::Hidden, cl::init(Val: true),
142 cl::Hidden, cl::desc("Use Stack Safety analysis results"));
143
144static cl::opt<size_t> ClMaxLifetimes(
145 "hwasan-max-lifetimes-for-alloca", cl::Hidden, cl::init(Val: 3),
146 cl::ReallyHidden,
147 cl::desc("How many lifetime ends to handle for a single alloca."));
148
149static cl::opt<bool>
150 ClUseAfterScope("hwasan-use-after-scope",
151 cl::desc("detect use after scope within function"),
152 cl::Hidden, cl::init(Val: true));
153
154static cl::opt<bool> ClStrictUseAfterScope(
155 "hwasan-strict-use-after-scope",
156 cl::desc("for complicated lifetimes, tag both on end and return"),
157 cl::Hidden, cl::init(Val: true));
158
159static cl::opt<bool> ClGenerateTagsWithCalls(
160 "hwasan-generate-tags-with-calls",
161 cl::desc("generate new tags with runtime library calls"), cl::Hidden,
162 cl::init(Val: false));
163
164static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
165 cl::Hidden, cl::init(Val: false));
166
167static cl::opt<bool> ClAllGlobals(
168 "hwasan-all-globals",
169 cl::desc(
170 "Instrument globals, even those within user-defined sections. Warning: "
171 "This may break existing code which walks globals via linker-generated "
172 "symbols, expects certain globals to be contiguous with each other, or "
173 "makes other assumptions which are invalidated by HWASan "
174 "instrumentation."),
175 cl::Hidden, cl::init(Val: false));
176
177static cl::opt<int> ClMatchAllTag(
178 "hwasan-match-all-tag",
179 cl::desc("don't report bad accesses via pointers with this tag"),
180 cl::Hidden, cl::init(Val: -1));
181
182static cl::opt<bool>
183 ClEnableKhwasan("hwasan-kernel",
184 cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
185 cl::Hidden, cl::init(Val: false));
186
187// These flags allow to change the shadow mapping and control how shadow memory
188// is accessed. The shadow mapping looks like:
189// Shadow = (Mem >> scale) + offset
190
191static cl::opt<uint64_t>
192 ClMappingOffset("hwasan-mapping-offset",
193 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
194 cl::Hidden);
195
196static cl::opt<OffsetKind> ClMappingOffsetDynamic(
197 "hwasan-mapping-offset-dynamic",
198 cl::desc("HWASan shadow mapping dynamic offset location"), cl::Hidden,
199 cl::values(clEnumValN(OffsetKind::kGlobal, "global", "Use global"),
200 clEnumValN(OffsetKind::kIfunc, "ifunc", "Use ifunc global"),
201 clEnumValN(OffsetKind::kTls, "tls", "Use TLS")));
202
203static cl::opt<bool>
204 ClFrameRecords("hwasan-with-frame-record",
205 cl::desc("Use ring buffer for stack allocations"),
206 cl::Hidden);
207
208static cl::opt<int> ClHotPercentileCutoff("hwasan-percentile-cutoff-hot",
209 cl::desc("Hot percentile cutoff."));
210
211static cl::opt<float>
212 ClRandomKeepRate("hwasan-random-rate",
213 cl::desc("Probability value in the range [0.0, 1.0] "
214 "to keep instrumentation of a function. "
215 "Note: instrumentation can be skipped randomly "
216 "OR because of the hot percentile cutoff, if "
217 "both are supplied."));
218
219static cl::opt<bool> ClStaticLinking(
220 "hwasan-static-linking",
221 cl::desc("Don't use .note.hwasan.globals section to instrument globals "
222 "from loadable libraries. "
223 "Note: in static binaries, the global variables section can be "
224 "accessed directly via linker-provided "
225 "__start_hwasan_globals and __stop_hwasan_globals symbols"),
226 cl::Hidden, cl::init(Val: false));
227
228// Mode for selecting how to insert frame record info into the stack ring
229// buffer.
230enum RecordStackHistoryMode {
231 // Do not record frame record info.
232 none,
233
234 // Insert instructions into the prologue for storing into the stack ring
235 // buffer directly.
236 instr,
237
238 // Add a call to __hwasan_add_frame_record in the runtime.
239 libcall,
240};
241
242static cl::opt<RecordStackHistoryMode> ClRecordStackHistory(
243 "hwasan-record-stack-history",
244 cl::desc("Record stack frames with tagged allocations in a thread-local "
245 "ring buffer"),
246 cl::values(clEnumVal(none, "Do not record stack ring history"),
247 clEnumVal(instr, "Insert instructions into the prologue for "
248 "storing into the stack ring buffer directly"),
249 clEnumVal(libcall, "Add a call to __hwasan_add_frame_record for "
250 "storing into the stack ring buffer")),
251 cl::Hidden, cl::init(Val: instr));
252
253static cl::opt<bool>
254 ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
255 cl::desc("instrument memory intrinsics"),
256 cl::Hidden, cl::init(Val: true));
257
258static cl::opt<bool>
259 ClInstrumentLandingPads("hwasan-instrument-landing-pads",
260 cl::desc("instrument landing pads"), cl::Hidden,
261 cl::init(Val: false));
262
263static cl::opt<bool> ClUseShortGranules(
264 "hwasan-use-short-granules",
265 cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
266 cl::init(Val: false));
267
268static cl::opt<bool> ClInstrumentPersonalityFunctions(
269 "hwasan-instrument-personality-functions",
270 cl::desc("instrument personality functions"), cl::Hidden);
271
272static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
273 cl::desc("inline all checks"),
274 cl::Hidden, cl::init(Val: false));
275
276static cl::opt<bool> ClInlineFastPathChecks("hwasan-inline-fast-path-checks",
277 cl::desc("inline all checks"),
278 cl::Hidden, cl::init(Val: false));
279
280// Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
281static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
282 cl::desc("Use page aliasing in HWASan"),
283 cl::Hidden, cl::init(Val: false));
284
285static cl::opt<uint64_t>
286 ClTagBits("hwasan-tag-bits",
287 cl::desc("Restrict tag to at most N bits. Needs to be > 4."),
288 cl::Hidden, cl::init(Val: 0));
289
290STATISTIC(NumTotalFuncs, "Number of total funcs");
291STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs");
292STATISTIC(NumNoProfileSummaryFuncs, "Number of funcs without PS");
293
294namespace {
295
296template <typename T> T optOr(cl::opt<T> &Opt, T Other) {
297 return Opt.getNumOccurrences() ? Opt : Other;
298}
299
300bool shouldUsePageAliases(const Triple &TargetTriple) {
301 return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64;
302}
303
304bool shouldInstrumentStack(const Triple &TargetTriple) {
305 return !shouldUsePageAliases(TargetTriple) && ClInstrumentStack;
306}
307
308bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
309 return optOr(Opt&: ClInstrumentWithCalls, Other: TargetTriple.getArch() == Triple::x86_64);
310}
311
312bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
313 return optOr(Opt&: ClUseStackSafety, Other: !DisableOptimization);
314}
315
316bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple,
317 bool DisableOptimization) {
318 return shouldInstrumentStack(TargetTriple) &&
319 mightUseStackSafetyAnalysis(DisableOptimization);
320}
321
322bool shouldDetectUseAfterScope(const Triple &TargetTriple) {
323 return ClUseAfterScope && shouldInstrumentStack(TargetTriple);
324}
325
326/// An instrumentation pass implementing detection of addressability bugs
327/// using tagged pointers.
328class HWAddressSanitizer {
329public:
330 HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover,
331 const StackSafetyGlobalInfo *SSI)
332 : M(M), SSI(SSI) {
333 this->Recover = optOr(Opt&: ClRecover, Other: Recover);
334 this->CompileKernel = optOr(Opt&: ClEnableKhwasan, Other: CompileKernel);
335 this->Rng = ClRandomKeepRate.getNumOccurrences() ? M.createRNG(DEBUG_TYPE)
336 : nullptr;
337
338 initializeModule();
339 }
340
341 void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
342
343private:
344 struct ShadowTagCheckInfo {
345 Instruction *TagMismatchTerm = nullptr;
346 Value *PtrLong = nullptr;
347 Value *AddrLong = nullptr;
348 Value *PtrTag = nullptr;
349 Value *MemTag = nullptr;
350 };
351
352 bool selectiveInstrumentationShouldSkip(Function &F,
353 FunctionAnalysisManager &FAM) const;
354 void initializeModule();
355 void createHwasanCtorComdat();
356 void createHwasanNote();
357
358 void initializeCallbacks(Module &M);
359
360 Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
361
362 Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
363 Value *getShadowNonTls(IRBuilder<> &IRB);
364
365 void untagPointerOperand(Instruction *I, Value *Addr);
366 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
367
368 int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
369 ShadowTagCheckInfo insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
370 DomTreeUpdater &DTU, LoopInfo *LI);
371 void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
372 unsigned AccessSizeIndex,
373 Instruction *InsertBefore,
374 DomTreeUpdater &DTU, LoopInfo *LI);
375 void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
376 unsigned AccessSizeIndex,
377 Instruction *InsertBefore, DomTreeUpdater &DTU,
378 LoopInfo *LI);
379 bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI);
380 void instrumentMemIntrinsic(MemIntrinsic *MI);
381 bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
382 LoopInfo *LI, const DataLayout &DL);
383 bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr);
384 bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst,
385 Value *Ptr);
386
387 void getInterestingMemoryOperands(
388 OptimizationRemarkEmitter &ORE, Instruction *I,
389 const TargetLibraryInfo &TLI,
390 SmallVectorImpl<InterestingMemoryOperand> &Interesting);
391
392 void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
393 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
394 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
395 void instrumentStack(OptimizationRemarkEmitter &ORE, memtag::StackInfo &Info,
396 Value *StackTag, Value *UARTag, const DominatorTree &DT,
397 const PostDominatorTree &PDT, const LoopInfo &LI);
398 void instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
399 Value *getNextTagWithCall(IRBuilder<> &IRB);
400 Value *getStackBaseTag(IRBuilder<> &IRB);
401 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
402 Value *getUARTag(IRBuilder<> &IRB);
403
404 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB);
405 Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
406 unsigned retagMask(unsigned AllocaNo);
407
408 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
409
410 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
411 void instrumentGlobals();
412
413 Value *getCachedFP(IRBuilder<> &IRB);
414 Value *getFrameRecordInfo(IRBuilder<> &IRB);
415
416 void instrumentPersonalityFunctions();
417
418 LLVMContext *C;
419 Module &M;
420 const StackSafetyGlobalInfo *SSI;
421 Triple TargetTriple;
422 std::unique_ptr<RandomNumberGenerator> Rng;
423
424 /// This struct defines the shadow mapping using the rule:
425 /// If `kFixed`, then
426 /// shadow = (mem >> Scale) + Offset.
427 /// If `kGlobal`, then
428 /// extern char* __hwasan_shadow_memory_dynamic_address;
429 /// shadow = (mem >> Scale) + __hwasan_shadow_memory_dynamic_address
430 /// If `kIfunc`, then
431 /// extern char __hwasan_shadow[];
432 /// shadow = (mem >> Scale) + &__hwasan_shadow
433 /// If `kTls`, then
434 /// extern char *__hwasan_tls;
435 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
436 ///
437 /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
438 /// ring buffer for storing stack allocations on targets that support it.
439 class ShadowMapping {
440 OffsetKind Kind;
441 uint64_t Offset;
442 uint8_t Scale;
443 bool WithFrameRecord;
444
445 void SetFixed(uint64_t O) {
446 Kind = OffsetKind::kFixed;
447 Offset = O;
448 }
449
450 public:
451 void init(Triple &TargetTriple, bool InstrumentWithCalls,
452 bool CompileKernel);
453 Align getObjectAlignment() const { return Align(1ULL << Scale); }
454 bool isInGlobal() const { return Kind == OffsetKind::kGlobal; }
455 bool isInIfunc() const { return Kind == OffsetKind::kIfunc; }
456 bool isInTls() const { return Kind == OffsetKind::kTls; }
457 bool isFixed() const { return Kind == OffsetKind::kFixed; }
458 uint8_t scale() const { return Scale; };
459 uint64_t offset() const {
460 assert(isFixed());
461 return Offset;
462 };
463 bool withFrameRecord() const { return WithFrameRecord; };
464 };
465
466 ShadowMapping Mapping;
467
468 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
469 Type *IntptrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
470 PointerType *PtrTy = PointerType::getUnqual(C&: M.getContext());
471 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
472 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
473 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
474
475 bool CompileKernel;
476 bool Recover;
477 bool OutlinedChecks;
478 bool InlineFastPath;
479 bool UseShortGranules;
480 bool InstrumentLandingPads;
481 bool InstrumentWithCalls;
482 bool InstrumentStack;
483 bool InstrumentGlobals;
484 bool DetectUseAfterScope;
485 bool UsePageAliases;
486 bool UseMatchAllCallback;
487
488 std::optional<uint8_t> MatchAllTag;
489
490 unsigned PointerTagShift;
491 uint64_t TagMaskByte;
492
493 Function *HwasanCtorFunction;
494
495 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
496 FunctionCallee HwasanMemoryAccessCallbackSized[2];
497
498 FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
499 FunctionCallee HwasanHandleVfork;
500
501 FunctionCallee HwasanTagMemoryFunc;
502 FunctionCallee HwasanGenerateTagFunc;
503 FunctionCallee HwasanRecordFrameRecordFunc;
504
505 Constant *ShadowGlobal;
506
507 Value *ShadowBase = nullptr;
508 Value *StackBaseTag = nullptr;
509 Value *CachedFP = nullptr;
510 GlobalValue *ThreadPtrGlobal = nullptr;
511};
512
513} // end anonymous namespace
514
515PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
516 ModuleAnalysisManager &MAM) {
517 // Return early if nosanitize_hwaddress module flag is present for the module.
518 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_hwaddress"))
519 return PreservedAnalyses::all();
520 const StackSafetyGlobalInfo *SSI = nullptr;
521 const Triple &TargetTriple = M.getTargetTriple();
522 if (shouldUseStackSafetyAnalysis(TargetTriple, DisableOptimization: Options.DisableOptimization))
523 SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M);
524
525 HWAddressSanitizer HWASan(M, Options.CompileKernel, Options.Recover, SSI);
526 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
527 for (Function &F : M)
528 HWASan.sanitizeFunction(F, FAM);
529
530 PreservedAnalyses PA = PreservedAnalyses::none();
531 // DominatorTreeAnalysis, PostDominatorTreeAnalysis, and LoopAnalysis
532 // are incrementally updated throughout this pass whenever
533 // SplitBlockAndInsertIfThen is called.
534 PA.preserve<DominatorTreeAnalysis>();
535 PA.preserve<PostDominatorTreeAnalysis>();
536 PA.preserve<LoopAnalysis>();
537 // GlobalsAA is considered stateless and does not get invalidated unless
538 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
539 // make changes that require GlobalsAA to be invalidated.
540 PA.abandon<GlobalsAA>();
541 return PA;
542}
543void HWAddressSanitizerPass::printPipeline(
544 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
545 static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
546 OS, MapClassName2PassName);
547 OS << '<';
548 if (Options.CompileKernel)
549 OS << "kernel;";
550 if (Options.Recover)
551 OS << "recover";
552 OS << '>';
553}
554
555void HWAddressSanitizer::createHwasanNote() {
556 // Create a note that contains pointers to the list of global
557 // descriptors. Adding a note to the output file will cause the linker to
558 // create a PT_NOTE program header pointing to the note that we can use to
559 // find the descriptor list starting from the program headers. A function
560 // provided by the runtime initializes the shadow memory for the globals by
561 // accessing the descriptor list via the note. The dynamic loader needs to
562 // call this function whenever a library is loaded.
563 //
564 // The reason why we use a note for this instead of a more conventional
565 // approach of having a global constructor pass a descriptor list pointer to
566 // the runtime is because of an order of initialization problem. With
567 // constructors we can encounter the following problematic scenario:
568 //
569 // 1) library A depends on library B and also interposes one of B's symbols
570 // 2) B's constructors are called before A's (as required for correctness)
571 // 3) during construction, B accesses one of its "own" globals (actually
572 // interposed by A) and triggers a HWASAN failure due to the initialization
573 // for A not having happened yet
574 //
575 // Even without interposition it is possible to run into similar situations in
576 // cases where two libraries mutually depend on each other.
577 //
578 // We only need one note per binary, so put everything for the note in a
579 // comdat. This needs to be a comdat with an .init_array section to prevent
580 // newer versions of lld from discarding the note.
581 //
582 // Create the note even if we aren't instrumenting globals. This ensures that
583 // binaries linked from object files with both instrumented and
584 // non-instrumented globals will end up with a note, even if a comdat from an
585 // object file with non-instrumented globals is selected. The note is harmless
586 // if the runtime doesn't support it, since it will just be ignored.
587 Comdat *NoteComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
588
589 Type *Int8Arr0Ty = ArrayType::get(ElementType: Int8Ty, NumElements: 0);
590 auto *Start =
591 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
592 nullptr, "__start_hwasan_globals");
593 Start->setVisibility(GlobalValue::HiddenVisibility);
594 auto *Stop =
595 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
596 nullptr, "__stop_hwasan_globals");
597 Stop->setVisibility(GlobalValue::HiddenVisibility);
598
599 // Null-terminated so actually 8 bytes, which are required in order to align
600 // the note properly.
601 auto *Name = ConstantDataArray::get(Context&: *C, Elts: "LLVM\0\0\0");
602
603 auto *NoteTy = StructType::get(elt1: Int32Ty, elts: Int32Ty, elts: Int32Ty, elts: Name->getType(),
604 elts: Int32Ty, elts: Int32Ty);
605 auto *Note =
606 new GlobalVariable(M, NoteTy, /*isConstant=*/true,
607 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
608 Note->setSection(".note.hwasan.globals");
609 Note->setComdat(NoteComdat);
610 Note->setAlignment(Align(4));
611
612 // The pointers in the note need to be relative so that the note ends up being
613 // placed in rodata, which is the standard location for notes.
614 auto CreateRelPtr = [&](Constant *Ptr) {
615 return ConstantExpr::getTrunc(
616 C: ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: Ptr, Ty: Int64Ty),
617 C2: ConstantExpr::getPtrToInt(C: Note, Ty: Int64Ty)),
618 Ty: Int32Ty);
619 };
620 Note->setInitializer(ConstantStruct::getAnon(
621 V: {ConstantInt::get(Ty: Int32Ty, V: 8), // n_namesz
622 ConstantInt::get(Ty: Int32Ty, V: 8), // n_descsz
623 ConstantInt::get(Ty: Int32Ty, V: ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
624 Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
625 appendToCompilerUsed(M, Values: Note);
626
627 // Create a zero-length global in hwasan_globals so that the linker will
628 // always create start and stop symbols.
629 auto *Dummy = new GlobalVariable(
630 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
631 Constant::getNullValue(Ty: Int8Arr0Ty), "hwasan.dummy.global");
632 Dummy->setSection("hwasan_globals");
633 Dummy->setComdat(NoteComdat);
634 Dummy->setMetadata(KindID: LLVMContext::MD_associated,
635 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: Note)));
636 appendToCompilerUsed(M, Values: Dummy);
637}
638
639void HWAddressSanitizer::createHwasanCtorComdat() {
640 std::tie(args&: HwasanCtorFunction, args: std::ignore) =
641 getOrCreateSanitizerCtorAndInitFunctions(
642 M, CtorName: kHwasanModuleCtorName, InitName: kHwasanInitName,
643 /*InitArgTypes=*/{},
644 /*InitArgs=*/{},
645 // This callback is invoked when the functions are created the first
646 // time. Hook them into the global ctors list in that case:
647 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
648 Comdat *CtorComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
649 Ctor->setComdat(CtorComdat);
650 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
651 });
652
653 // Do not create .note.hwasan.globals for static binaries, as it is only
654 // needed for instrumenting globals from dynamic libraries. In static
655 // binaries, the global variables section can be accessed directly via the
656 // __start_hwasan_globals and __stop_hwasan_globals symbols inserted by the
657 // linker.
658 if (!ClStaticLinking)
659 createHwasanNote();
660}
661
662/// Module-level initialization.
663///
664/// inserts a call to __hwasan_init to the module's constructor list.
665void HWAddressSanitizer::initializeModule() {
666 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
667 TargetTriple = M.getTargetTriple();
668
669 // HWASan may do short granule checks on function arguments read from the
670 // argument memory (last byte of the granule), which invalidates writeonly.
671 for (Function &F : M.functions())
672 removeASanIncompatibleFnAttributes(F, /*ReadsArgMem=*/true);
673
674 // x86_64 currently has two modes:
675 // - Intel LAM (default)
676 // - pointer aliasing (heap only)
677 bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
678 UsePageAliases = shouldUsePageAliases(TargetTriple);
679 InstrumentWithCalls = shouldInstrumentWithCalls(TargetTriple);
680 InstrumentStack = shouldInstrumentStack(TargetTriple);
681 DetectUseAfterScope = shouldDetectUseAfterScope(TargetTriple);
682 PointerTagShift = IsX86_64 ? 57 : 56;
683 TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
684 if (ClTagBits) {
685 if (TagMaskByte < 4)
686 reportFatalUsageError(
687 reason: "need more than 4 bits of tag to have non-short-granule tags");
688 TagMaskByte &= (1ULL << ClTagBits) - 1;
689 }
690
691 Mapping.init(TargetTriple, InstrumentWithCalls, CompileKernel);
692
693 C = &(M.getContext());
694 IRBuilder<> IRB(M);
695
696 HwasanCtorFunction = nullptr;
697
698 // Older versions of Android do not have the required runtime support for
699 // short granules, global or personality function instrumentation. On other
700 // platforms we currently require using the latest version of the runtime.
701 bool NewRuntime =
702 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(Major: 30);
703
704 UseShortGranules = optOr(Opt&: ClUseShortGranules, Other: NewRuntime);
705 OutlinedChecks = (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
706 TargetTriple.isOSBinFormatELF() &&
707 !optOr(Opt&: ClInlineAllChecks, Other: Recover);
708
709 // These platforms may prefer less inlining to reduce binary size.
710 InlineFastPath = optOr(Opt&: ClInlineFastPathChecks, Other: !(TargetTriple.isAndroid() ||
711 TargetTriple.isOSFuchsia()));
712
713 if (ClMatchAllTag.getNumOccurrences()) {
714 if (ClMatchAllTag != -1) {
715 MatchAllTag = ClMatchAllTag & 0xFF;
716 }
717 } else if (CompileKernel) {
718 MatchAllTag = 0xFF;
719 }
720 UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
721
722 // If we don't have personality function support, fall back to landing pads.
723 InstrumentLandingPads = optOr(Opt&: ClInstrumentLandingPads, Other: !NewRuntime);
724
725 InstrumentGlobals =
726 !CompileKernel && !UsePageAliases && optOr(Opt&: ClGlobals, Other: NewRuntime);
727
728 if (!CompileKernel) {
729 if (InstrumentGlobals)
730 instrumentGlobals();
731
732 createHwasanCtorComdat();
733
734 bool InstrumentPersonalityFunctions =
735 optOr(Opt&: ClInstrumentPersonalityFunctions, Other: NewRuntime);
736 if (InstrumentPersonalityFunctions)
737 instrumentPersonalityFunctions();
738 }
739
740 if (!TargetTriple.isAndroid()) {
741 ThreadPtrGlobal = M.getOrInsertGlobal(Name: "__hwasan_tls", Ty: IntptrTy, CreateGlobalCallback: [&] {
742 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
743 GlobalValue::ExternalLinkage, nullptr,
744 "__hwasan_tls", nullptr,
745 GlobalVariable::InitialExecTLSModel);
746 appendToCompilerUsed(M, Values: GV);
747 return GV;
748 });
749 }
750}
751
752void HWAddressSanitizer::initializeCallbacks(Module &M) {
753 IRBuilder<> IRB(M);
754 const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
755 FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
756 *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
757 *HwasanMemsetFnTy;
758 if (UseMatchAllCallback) {
759 HwasanMemoryAccessCallbackSizedFnTy =
760 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy, Int8Ty}, isVarArg: false);
761 HwasanMemoryAccessCallbackFnTy =
762 FunctionType::get(Result: VoidTy, Params: {IntptrTy, Int8Ty}, isVarArg: false);
763 HwasanMemTransferFnTy =
764 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy, Int8Ty}, isVarArg: false);
765 HwasanMemsetFnTy =
766 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy, Int8Ty}, isVarArg: false);
767 } else {
768 HwasanMemoryAccessCallbackSizedFnTy =
769 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy}, isVarArg: false);
770 HwasanMemoryAccessCallbackFnTy =
771 FunctionType::get(Result: VoidTy, Params: {IntptrTy}, isVarArg: false);
772 HwasanMemTransferFnTy =
773 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy}, isVarArg: false);
774 HwasanMemsetFnTy =
775 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy}, isVarArg: false);
776 }
777
778 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
779 const std::string TypeStr = AccessIsWrite ? "store" : "load";
780 const std::string EndingStr = Recover ? "_noabort" : "";
781
782 HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
783 Name: ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
784 T: HwasanMemoryAccessCallbackSizedFnTy);
785
786 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
787 AccessSizeIndex++) {
788 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
789 M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + TypeStr +
790 itostr(X: 1ULL << AccessSizeIndex) +
791 MatchAllStr + EndingStr,
792 T: HwasanMemoryAccessCallbackFnTy);
793 }
794 }
795
796 const std::string MemIntrinCallbackPrefix =
797 (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
798 ? std::string("")
799 : ClMemoryAccessCallbackPrefix;
800
801 HwasanMemmove = M.getOrInsertFunction(
802 Name: MemIntrinCallbackPrefix + "memmove" + MatchAllStr, T: HwasanMemTransferFnTy);
803 HwasanMemcpy = M.getOrInsertFunction(
804 Name: MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, T: HwasanMemTransferFnTy);
805 HwasanMemset = M.getOrInsertFunction(
806 Name: MemIntrinCallbackPrefix + "memset" + MatchAllStr, T: HwasanMemsetFnTy);
807
808 HwasanTagMemoryFunc = M.getOrInsertFunction(Name: "__hwasan_tag_memory", RetTy: VoidTy,
809 Args: PtrTy, Args: Int8Ty, Args: IntptrTy);
810 HwasanGenerateTagFunc =
811 M.getOrInsertFunction(Name: "__hwasan_generate_tag", RetTy: Int8Ty);
812
813 HwasanRecordFrameRecordFunc =
814 M.getOrInsertFunction(Name: "__hwasan_add_frame_record", RetTy: VoidTy, Args: Int64Ty);
815
816 ShadowGlobal =
817 M.getOrInsertGlobal(Name: "__hwasan_shadow", Ty: ArrayType::get(ElementType: Int8Ty, NumElements: 0));
818
819 HwasanHandleVfork =
820 M.getOrInsertFunction(Name: "__hwasan_handle_vfork", RetTy: VoidTy, Args: IntptrTy);
821}
822
823Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
824 // An empty inline asm with input reg == output reg.
825 // An opaque no-op cast, basically.
826 // This prevents code bloat as a result of rematerializing trivial definitions
827 // such as constants or global addresses at every load and store.
828 InlineAsm *Asm =
829 InlineAsm::get(Ty: FunctionType::get(Result: PtrTy, Params: {Val->getType()}, isVarArg: false),
830 AsmString: StringRef(""), Constraints: StringRef("=r,0"),
831 /*hasSideEffects=*/false);
832 return IRB.CreateCall(Callee: Asm, Args: {Val}, Name: ".hwasan.shadow");
833}
834
835Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
836 return getOpaqueNoopCast(IRB, Val: ShadowGlobal);
837}
838
839Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
840 if (Mapping.isFixed()) {
841 return getOpaqueNoopCast(
842 IRB, Val: ConstantExpr::getIntToPtr(
843 C: ConstantInt::get(Ty: IntptrTy, V: Mapping.offset()), Ty: PtrTy));
844 }
845
846 if (Mapping.isInIfunc())
847 return getDynamicShadowIfunc(IRB);
848
849 Value *GlobalDynamicAddress = IRB.getModule()->getOrInsertGlobal(
850 Name: kHwasanShadowMemoryDynamicAddress, Ty: PtrTy);
851 return IRB.CreateLoad(Ty: PtrTy, Ptr: GlobalDynamicAddress);
852}
853
854bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst,
855 Value *Ptr) {
856 // Do not instrument accesses from different address spaces; we cannot deal
857 // with them.
858 Type *PtrTy = cast<PointerType>(Val: Ptr->getType()->getScalarType());
859 if (PtrTy->getPointerAddressSpace() != 0)
860 return true;
861
862 // Ignore swifterror addresses.
863 // swifterror memory addresses are mem2reg promoted by instruction
864 // selection. As such they cannot have regular uses like an instrumentation
865 // function and it makes no sense to track them as memory.
866 if (Ptr->isSwiftError())
867 return true;
868
869 if (findAllocaForValue(V: Ptr)) {
870 if (!InstrumentStack)
871 return true;
872 if (SSI && SSI->stackAccessIsSafe(I: *Inst))
873 return true;
874 }
875
876 if (isa<GlobalVariable>(Val: getUnderlyingObject(V: Ptr))) {
877 if (!InstrumentGlobals)
878 return true;
879 // TODO: Optimize inbound global accesses, like Asan `instrumentMop`.
880 }
881
882 return false;
883}
884
885bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE,
886 Instruction *Inst, Value *Ptr) {
887 bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr);
888 if (Ignored) {
889 ORE.emit(
890 RemarkBuilder: [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); });
891 } else {
892 ORE.emit(RemarkBuilder: [&]() {
893 return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst);
894 });
895 }
896 return Ignored;
897}
898
899void HWAddressSanitizer::getInterestingMemoryOperands(
900 OptimizationRemarkEmitter &ORE, Instruction *I,
901 const TargetLibraryInfo &TLI,
902 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
903 // Skip memory accesses inserted by another instrumentation.
904 if (I->hasMetadata(KindID: LLVMContext::MD_nosanitize))
905 return;
906
907 // Do not instrument the load fetching the dynamic shadow address.
908 if (ShadowBase == I)
909 return;
910
911 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
912 if (!ClInstrumentReads || ignoreAccess(ORE, Inst: I, Ptr: LI->getPointerOperand()))
913 return;
914 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
915 Args: LI->getType(), Args: LI->getAlign());
916 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
917 if (!ClInstrumentWrites || ignoreAccess(ORE, Inst: I, Ptr: SI->getPointerOperand()))
918 return;
919 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
920 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
921 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
922 if (!ClInstrumentAtomics || ignoreAccess(ORE, Inst: I, Ptr: RMW->getPointerOperand()))
923 return;
924 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
925 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
926 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
927 if (!ClInstrumentAtomics || ignoreAccess(ORE, Inst: I, Ptr: XCHG->getPointerOperand()))
928 return;
929 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
930 Args: XCHG->getCompareOperand()->getType(),
931 Args: std::nullopt);
932 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
933 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
934 if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
935 ignoreAccess(ORE, Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
936 continue;
937 Type *Ty = CI->getParamByValType(ArgNo);
938 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
939 }
940 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
941 }
942}
943
944static unsigned getPointerOperandIndex(Instruction *I) {
945 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I))
946 return LI->getPointerOperandIndex();
947 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I))
948 return SI->getPointerOperandIndex();
949 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I))
950 return RMW->getPointerOperandIndex();
951 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I))
952 return XCHG->getPointerOperandIndex();
953 report_fatal_error(reason: "Unexpected instruction");
954 return -1;
955}
956
957static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
958 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
959 assert(Res < kNumberOfAccessSizes);
960 return Res;
961}
962
963void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
964 if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
965 TargetTriple.isRISCV64())
966 return;
967
968 IRBuilder<> IRB(I);
969 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
970 Value *UntaggedPtr =
971 IRB.CreateIntToPtr(V: untagPointer(IRB, PtrLong: AddrLong), DestTy: Addr->getType());
972 I->setOperand(i: getPointerOperandIndex(I), Val: UntaggedPtr);
973}
974
975Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
976 // Mem >> Scale
977 Value *Shadow = IRB.CreateLShr(LHS: Mem, RHS: Mapping.scale());
978 if (Mapping.isFixed() && Mapping.offset() == 0)
979 return IRB.CreateIntToPtr(V: Shadow, DestTy: PtrTy);
980 // (Mem >> Scale) + Offset
981 return IRB.CreatePtrAdd(Ptr: ShadowBase, Offset: Shadow);
982}
983
984int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
985 unsigned AccessSizeIndex) {
986 return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
987 (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
988 (MatchAllTag.value_or(u: 0) << HWASanAccessInfo::MatchAllShift) |
989 (Recover << HWASanAccessInfo::RecoverShift) |
990 (IsWrite << HWASanAccessInfo::IsWriteShift) |
991 (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
992}
993
994HWAddressSanitizer::ShadowTagCheckInfo
995HWAddressSanitizer::insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
996 DomTreeUpdater &DTU, LoopInfo *LI) {
997 ShadowTagCheckInfo R;
998
999 IRBuilder<> IRB(InsertBefore);
1000
1001 R.PtrLong = IRB.CreatePointerCast(V: Ptr, DestTy: IntptrTy);
1002 R.PtrTag =
1003 IRB.CreateTrunc(V: IRB.CreateLShr(LHS: R.PtrLong, RHS: PointerTagShift), DestTy: Int8Ty);
1004 R.AddrLong = untagPointer(IRB, PtrLong: R.PtrLong);
1005 Value *Shadow = memToShadow(Mem: R.AddrLong, IRB);
1006 R.MemTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: Shadow);
1007 Value *TagMismatch = IRB.CreateICmpNE(LHS: R.PtrTag, RHS: R.MemTag);
1008
1009 if (MatchAllTag.has_value()) {
1010 Value *TagNotIgnored = IRB.CreateICmpNE(
1011 LHS: R.PtrTag, RHS: ConstantInt::get(Ty: R.PtrTag->getType(), V: *MatchAllTag));
1012 TagMismatch = IRB.CreateAnd(LHS: TagMismatch, RHS: TagNotIgnored);
1013 }
1014
1015 R.TagMismatchTerm = SplitBlockAndInsertIfThen(
1016 Cond: TagMismatch, SplitBefore: InsertBefore, Unreachable: false,
1017 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
1018
1019 return R;
1020}
1021
1022void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
1023 unsigned AccessSizeIndex,
1024 Instruction *InsertBefore,
1025 DomTreeUpdater &DTU,
1026 LoopInfo *LI) {
1027 assert(!UsePageAliases);
1028 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
1029
1030 if (InlineFastPath)
1031 InsertBefore =
1032 insertShadowTagCheck(Ptr, InsertBefore, DTU, LI).TagMismatchTerm;
1033
1034 IRBuilder<> IRB(InsertBefore);
1035 bool UseFixedShadowIntrinsic = false;
1036 // The memaccess fixed shadow intrinsic is only supported on AArch64,
1037 // which allows a 16-bit immediate to be left-shifted by 32.
1038 // Since kShadowBaseAlignment == 32, and Linux by default will not
1039 // mmap above 48-bits, practically any valid shadow offset is
1040 // representable.
1041 // In particular, an offset of 4TB (1024 << 32) is representable, and
1042 // ought to be good enough for anybody.
1043 if (TargetTriple.isAArch64() && Mapping.isFixed()) {
1044 uint16_t OffsetShifted = Mapping.offset() >> 32;
1045 UseFixedShadowIntrinsic =
1046 static_cast<uint64_t>(OffsetShifted) << 32 == Mapping.offset();
1047 }
1048
1049 if (UseFixedShadowIntrinsic) {
1050 IRB.CreateIntrinsic(
1051 ID: UseShortGranules
1052 ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
1053 : Intrinsic::hwasan_check_memaccess_fixedshadow,
1054 Args: {Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo),
1055 ConstantInt::get(Ty: Int64Ty, V: Mapping.offset())});
1056 } else {
1057 IRB.CreateIntrinsic(
1058 ID: UseShortGranules ? Intrinsic::hwasan_check_memaccess_shortgranules
1059 : Intrinsic::hwasan_check_memaccess,
1060 Args: {ShadowBase, Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo)});
1061 }
1062}
1063
1064void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
1065 unsigned AccessSizeIndex,
1066 Instruction *InsertBefore,
1067 DomTreeUpdater &DTU,
1068 LoopInfo *LI) {
1069 assert(!UsePageAliases);
1070 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
1071
1072 ShadowTagCheckInfo TCI = insertShadowTagCheck(Ptr, InsertBefore, DTU, LI);
1073
1074 IRBuilder<> IRB(TCI.TagMismatchTerm);
1075 Value *OutOfShortGranuleTagRange =
1076 IRB.CreateICmpUGT(LHS: TCI.MemTag, RHS: ConstantInt::get(Ty: Int8Ty, V: 15));
1077 Instruction *CheckFailTerm = SplitBlockAndInsertIfThen(
1078 Cond: OutOfShortGranuleTagRange, SplitBefore: TCI.TagMismatchTerm, Unreachable: !Recover,
1079 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
1080
1081 IRB.SetInsertPoint(TCI.TagMismatchTerm);
1082 Value *PtrLowBits = IRB.CreateTrunc(V: IRB.CreateAnd(LHS: TCI.PtrLong, RHS: 15), DestTy: Int8Ty);
1083 PtrLowBits = IRB.CreateAdd(
1084 LHS: PtrLowBits, RHS: ConstantInt::get(Ty: Int8Ty, V: (1 << AccessSizeIndex) - 1));
1085 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(LHS: PtrLowBits, RHS: TCI.MemTag);
1086 SplitBlockAndInsertIfThen(Cond: PtrLowBitsOOB, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
1087 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
1088 LI, ThenBlock: CheckFailTerm->getParent());
1089
1090 IRB.SetInsertPoint(TCI.TagMismatchTerm);
1091 Value *InlineTagAddr = IRB.CreateOr(LHS: TCI.AddrLong, RHS: 15);
1092 InlineTagAddr = IRB.CreateIntToPtr(V: InlineTagAddr, DestTy: PtrTy);
1093 Value *InlineTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: InlineTagAddr);
1094 Value *InlineTagMismatch = IRB.CreateICmpNE(LHS: TCI.PtrTag, RHS: InlineTag);
1095 SplitBlockAndInsertIfThen(Cond: InlineTagMismatch, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
1096 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
1097 LI, ThenBlock: CheckFailTerm->getParent());
1098
1099 IRB.SetInsertPoint(CheckFailTerm);
1100 InlineAsm *Asm;
1101 switch (TargetTriple.getArch()) {
1102 case Triple::x86_64:
1103 // The signal handler will find the data address in rdi.
1104 Asm = InlineAsm::get(
1105 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1106 AsmString: "int3\nnopl " +
1107 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
1108 "(%rax)",
1109 Constraints: "{rdi}",
1110 /*hasSideEffects=*/true);
1111 break;
1112 case Triple::aarch64:
1113 case Triple::aarch64_be:
1114 // The signal handler will find the data address in x0.
1115 Asm = InlineAsm::get(
1116 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1117 AsmString: "brk #" + itostr(X: 0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
1118 Constraints: "{x0}",
1119 /*hasSideEffects=*/true);
1120 break;
1121 case Triple::riscv64:
1122 // The signal handler will find the data address in x10.
1123 Asm = InlineAsm::get(
1124 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
1125 AsmString: "ebreak\naddiw x0, x11, " +
1126 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
1127 Constraints: "{x10}",
1128 /*hasSideEffects=*/true);
1129 break;
1130 default:
1131 report_fatal_error(reason: "unsupported architecture");
1132 }
1133 IRB.CreateCall(Callee: Asm, Args: TCI.PtrLong);
1134 if (Recover)
1135 cast<UncondBrInst>(Val: CheckFailTerm)
1136 ->setSuccessor(TCI.TagMismatchTerm->getParent());
1137}
1138
1139bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE,
1140 MemIntrinsic *MI) {
1141 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Val: MI)) {
1142 return (!ClInstrumentWrites || ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getDest())) &&
1143 (!ClInstrumentReads || ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getSource()));
1144 }
1145 if (isa<MemSetInst>(Val: MI))
1146 return !ClInstrumentWrites || ignoreAccess(ORE, Inst: MI, Ptr: MI->getDest());
1147 return false;
1148}
1149
1150void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
1151 IRBuilder<> IRB(MI);
1152 if (isa<MemTransferInst>(Val: MI)) {
1153 SmallVector<Value *, 4> Args{
1154 MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
1155 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1156
1157 if (UseMatchAllCallback)
1158 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1159 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? HwasanMemmove : HwasanMemcpy, Args);
1160 } else if (isa<MemSetInst>(Val: MI)) {
1161 SmallVector<Value *, 4> Args{
1162 MI->getOperand(i_nocapture: 0),
1163 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1164 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1165 if (UseMatchAllCallback)
1166 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1167 IRB.CreateCall(Callee: HwasanMemset, Args);
1168 }
1169 MI->eraseFromParent();
1170}
1171
1172bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O,
1173 DomTreeUpdater &DTU, LoopInfo *LI,
1174 const DataLayout &DL) {
1175 Value *Addr = O.getPtr();
1176
1177 LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
1178
1179 // If the pointer is statically known to be zero, the tag check will pass
1180 // since:
1181 // 1) it has a zero tag
1182 // 2) the shadow memory corresponding to address 0 is initialized to zero and
1183 // never updated.
1184 // We can therefore elide the tag check.
1185 llvm::KnownBits Known(DL.getPointerTypeSizeInBits(Addr->getType()));
1186 llvm::computeKnownBits(V: Addr, Known, DL);
1187 if (Known.isZero())
1188 return false;
1189
1190 if (O.MaybeMask)
1191 return false; // FIXME
1192
1193 IRBuilder<> IRB(O.getInsn());
1194 if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(Value: O.TypeStoreSize) &&
1195 (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1196 (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
1197 *O.Alignment >= O.TypeStoreSize / 8)) {
1198 size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize: O.TypeStoreSize);
1199 if (InstrumentWithCalls) {
1200 SmallVector<Value *, 2> Args{IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy)};
1201 if (UseMatchAllCallback)
1202 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1203 IRB.CreateCall(Callee: HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1204 Args);
1205 } else if (OutlinedChecks) {
1206 instrumentMemAccessOutline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1207 DTU, LI);
1208 } else {
1209 instrumentMemAccessInline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1210 DTU, LI);
1211 }
1212 } else {
1213 SmallVector<Value *, 3> Args{
1214 IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy),
1215 IRB.CreateUDiv(LHS: IRB.CreateTypeSize(Ty: IntptrTy, Size: O.TypeStoreSize),
1216 RHS: ConstantInt::get(Ty: IntptrTy, V: 8))};
1217 if (UseMatchAllCallback)
1218 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1219 IRB.CreateCall(Callee: HwasanMemoryAccessCallbackSized[O.IsWrite], Args);
1220 }
1221 untagPointerOperand(I: O.getInsn(), Addr);
1222
1223 return true;
1224}
1225
1226void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1227 size_t Size) {
1228 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1229 if (!UseShortGranules)
1230 Size = AlignedSize;
1231
1232 Tag = IRB.CreateTrunc(V: Tag, DestTy: Int8Ty);
1233 if (InstrumentWithCalls) {
1234 IRB.CreateCall(Callee: HwasanTagMemoryFunc,
1235 Args: {IRB.CreatePointerCast(V: AI, DestTy: PtrTy), Tag,
1236 ConstantInt::get(Ty: IntptrTy, V: AlignedSize)});
1237 } else {
1238 size_t ShadowSize = Size >> Mapping.scale();
1239 Value *AddrLong = untagPointer(IRB, PtrLong: IRB.CreatePointerCast(V: AI, DestTy: IntptrTy));
1240 Value *ShadowPtr = memToShadow(Mem: AddrLong, IRB);
1241 // If this memset is not inlined, it will be intercepted in the hwasan
1242 // runtime library. That's OK, because the interceptor skips the checks if
1243 // the address is in the shadow region.
1244 // FIXME: the interceptor is not as fast as real memset. Consider lowering
1245 // llvm.memset right here into either a sequence of stores, or a call to
1246 // hwasan_tag_memory.
1247 if (ShadowSize)
1248 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Tag, Size: ShadowSize, Align: Align(1));
1249 if (Size != AlignedSize) {
1250 const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1251 IRB.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: SizeRemainder),
1252 Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: ShadowPtr, Idx0: ShadowSize));
1253 IRB.CreateStore(
1254 Val: Tag, Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: IRB.CreatePointerCast(V: AI, DestTy: PtrTy),
1255 Idx0: AlignedSize - 1));
1256 }
1257 }
1258}
1259
1260unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1261 if (TargetTriple.getArch() == Triple::x86_64)
1262 return AllocaNo & TagMaskByte;
1263
1264 // A list of 8-bit numbers that have at most one run of non-zero bits.
1265 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1266 // masks.
1267 // The list does not include the value 255, which is used for UAR.
1268 //
1269 // Because we are more likely to use earlier elements of this list than later
1270 // ones, it is sorted in increasing order of probability of collision with a
1271 // mask allocated (temporally) nearby. The program that generated this list
1272 // can be found at:
1273 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1274 static const unsigned FastMasks[] = {
1275 0, 128, 64, 192, 32, 96, 224, 112, 240, 48, 16, 120,
1276 248, 56, 24, 8, 124, 252, 60, 28, 12, 4, 126, 254,
1277 62, 30, 14, 6, 2, 127, 63, 31, 15, 7, 3, 1};
1278 return FastMasks[AllocaNo % std::size(FastMasks)];
1279}
1280
1281Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1282 if (TagMaskByte == 0xFF)
1283 return OldTag; // No need to clear the tag byte.
1284 return IRB.CreateAnd(LHS: OldTag,
1285 RHS: ConstantInt::get(Ty: OldTag->getType(), V: TagMaskByte));
1286}
1287
1288Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1289 return IRB.CreateZExt(V: IRB.CreateCall(Callee: HwasanGenerateTagFunc), DestTy: IntptrTy);
1290}
1291
1292Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1293 if (ClGenerateTagsWithCalls)
1294 return nullptr;
1295 if (StackBaseTag)
1296 return StackBaseTag;
1297 // Extract some entropy from the stack pointer for the tags.
1298 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1299 // between functions).
1300 Value *FramePointerLong = getCachedFP(IRB);
1301 Value *StackTag =
1302 applyTagMask(IRB, OldTag: IRB.CreateXor(LHS: FramePointerLong,
1303 RHS: IRB.CreateLShr(LHS: FramePointerLong, RHS: 20)));
1304 StackTag->setName("hwasan.stack.base.tag");
1305 return StackTag;
1306}
1307
1308Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1309 unsigned AllocaNo) {
1310 if (ClGenerateTagsWithCalls)
1311 return getNextTagWithCall(IRB);
1312 return IRB.CreateXor(
1313 LHS: StackTag, RHS: ConstantInt::get(Ty: StackTag->getType(), V: retagMask(AllocaNo)));
1314}
1315
1316Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
1317 Value *FramePointerLong = getCachedFP(IRB);
1318 Value *UARTag =
1319 applyTagMask(IRB, OldTag: IRB.CreateLShr(LHS: FramePointerLong, RHS: PointerTagShift));
1320
1321 UARTag->setName("hwasan.uar.tag");
1322 return UARTag;
1323}
1324
1325// Add a tag to an address.
1326Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1327 Value *PtrLong, Value *Tag) {
1328 assert(!UsePageAliases);
1329 Value *TaggedPtrLong;
1330 if (CompileKernel) {
1331 // Kernel addresses have 0xFF in the most significant byte.
1332 Value *ShiftedTag =
1333 IRB.CreateOr(LHS: IRB.CreateShl(LHS: Tag, RHS: PointerTagShift),
1334 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << PointerTagShift) - 1));
1335 TaggedPtrLong = IRB.CreateAnd(LHS: PtrLong, RHS: ShiftedTag);
1336 } else {
1337 // Userspace can simply do OR (tag << PointerTagShift);
1338 Value *ShiftedTag = IRB.CreateShl(LHS: Tag, RHS: PointerTagShift);
1339 TaggedPtrLong = IRB.CreateOr(LHS: PtrLong, RHS: ShiftedTag);
1340 }
1341 return IRB.CreateIntToPtr(V: TaggedPtrLong, DestTy: Ty);
1342}
1343
1344// Remove tag from an address.
1345Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1346 assert(!UsePageAliases);
1347 Value *UntaggedPtrLong;
1348 if (CompileKernel) {
1349 // Kernel addresses have 0xFF in the most significant byte.
1350 UntaggedPtrLong =
1351 IRB.CreateOr(LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1352 V: TagMaskByte << PointerTagShift));
1353 } else {
1354 // Userspace addresses have 0x00.
1355 UntaggedPtrLong = IRB.CreateAnd(
1356 LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1357 V: ~(TagMaskByte << PointerTagShift)));
1358 }
1359 return UntaggedPtrLong;
1360}
1361
1362Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) {
1363 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1364 // in Bionic's libc/platform/bionic/tls_defines.h.
1365 constexpr int SanitizerSlot = 6;
1366 if (TargetTriple.isAArch64() && TargetTriple.isAndroid())
1367 return memtag::getAndroidSlotPtr(IRB, Slot: SanitizerSlot);
1368 return ThreadPtrGlobal;
1369}
1370
1371Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) {
1372 if (!CachedFP)
1373 CachedFP = memtag::getFP(IRB);
1374 return CachedFP;
1375}
1376
1377Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1378 // Prepare ring buffer data.
1379 Value *PC = memtag::getPC(TargetTriple, IRB);
1380 Value *FP = getCachedFP(IRB);
1381
1382 // Mix FP and PC.
1383 // Assumptions:
1384 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
1385 // FP is 0xfffffffffffFFFF0 (4 lower bits are zero)
1386 // We only really need ~20 lower non-zero bits (FFFF), so we mix like this:
1387 // 0xFFFFPPPPPPPPPPPP
1388 //
1389 // FP works because in AArch64FrameLowering::getFrameIndexReference, we
1390 // prefer FP-relative offsets for functions compiled with HWASan.
1391 FP = IRB.CreateShl(LHS: FP, RHS: 44);
1392 return IRB.CreateOr(LHS: PC, RHS: FP);
1393}
1394
1395void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1396 if (!Mapping.isInTls())
1397 ShadowBase = getShadowNonTls(IRB);
1398 else if (!WithFrameRecord && TargetTriple.isAndroid())
1399 ShadowBase = getDynamicShadowIfunc(IRB);
1400
1401 if (!WithFrameRecord && ShadowBase)
1402 return;
1403
1404 Value *SlotPtr = nullptr;
1405 Value *ThreadLong = nullptr;
1406 Value *ThreadLongMaybeUntagged = nullptr;
1407
1408 auto getThreadLongMaybeUntagged = [&]() {
1409 if (!SlotPtr)
1410 SlotPtr = getHwasanThreadSlotPtr(IRB);
1411 if (!ThreadLong)
1412 ThreadLong = IRB.CreateLoad(Ty: IntptrTy, Ptr: SlotPtr);
1413 // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1414 // TBI.
1415 return TargetTriple.isAArch64() ? ThreadLong
1416 : untagPointer(IRB, PtrLong: ThreadLong);
1417 };
1418
1419 if (WithFrameRecord) {
1420 switch (ClRecordStackHistory) {
1421 case libcall: {
1422 // Emit a runtime call into hwasan rather than emitting instructions for
1423 // recording stack history.
1424 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1425 IRB.CreateCall(Callee: HwasanRecordFrameRecordFunc, Args: {FrameRecordInfo});
1426 break;
1427 }
1428 case instr: {
1429 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1430
1431 StackBaseTag = IRB.CreateAShr(LHS: ThreadLong, RHS: 3);
1432
1433 // Store data to ring buffer.
1434 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1435 Value *RecordPtr =
1436 IRB.CreateIntToPtr(V: ThreadLongMaybeUntagged, DestTy: IRB.getPtrTy(AddrSpace: 0));
1437 IRB.CreateStore(Val: FrameRecordInfo, Ptr: RecordPtr);
1438
1439 IRB.CreateStore(Val: memtag::incrementThreadLong(IRB, ThreadLong, Inc: 8), Ptr: SlotPtr);
1440 break;
1441 }
1442 case none: {
1443 llvm_unreachable(
1444 "A stack history recording mode should've been selected.");
1445 }
1446 }
1447 }
1448
1449 if (!ShadowBase) {
1450 if (!ThreadLongMaybeUntagged)
1451 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1452
1453 // Get shadow base address by aligning RecordPtr up.
1454 // Note: this is not correct if the pointer is already aligned.
1455 // Runtime library will make sure this never happens.
1456 ShadowBase = IRB.CreateAdd(
1457 LHS: IRB.CreateOr(
1458 LHS: ThreadLongMaybeUntagged,
1459 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << kShadowBaseAlignment) - 1)),
1460 RHS: ConstantInt::get(Ty: IntptrTy, V: 1), Name: "hwasan.shadow");
1461 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: PtrTy);
1462 }
1463}
1464
1465void HWAddressSanitizer::instrumentLandingPads(
1466 SmallVectorImpl<Instruction *> &LandingPadVec) {
1467 for (auto *LP : LandingPadVec) {
1468 IRBuilder<> IRB(LP->getNextNode());
1469 IRB.CreateCall(
1470 Callee: HwasanHandleVfork,
1471 Args: {memtag::readRegister(
1472 IRB, Name: (TargetTriple.getArch() == Triple::x86_64) ? "rsp" : "sp")});
1473 }
1474}
1475
1476void HWAddressSanitizer::instrumentStack(OptimizationRemarkEmitter &ORE,
1477 memtag::StackInfo &SInfo,
1478 Value *StackTag, Value *UARTag,
1479 const DominatorTree &DT,
1480 const PostDominatorTree &PDT,
1481 const LoopInfo &LI) {
1482 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1483 // alloca addresses using that. Unfortunately, offsets are not known yet
1484 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1485 // temp, shift-OR it into each alloca address and xor with the retag mask.
1486 // This generates one extra instruction per alloca use.
1487 unsigned int I = 0;
1488
1489 for (auto &KV : SInfo.AllocasToInstrument) {
1490 auto N = I++;
1491 auto *AI = KV.first;
1492 memtag::AllocaInfo &Info = KV.second;
1493 IRBuilder<> IRB(AI->getNextNode());
1494
1495 // Replace uses of the alloca with tagged address.
1496 Value *Tag = getAllocaTag(IRB, StackTag, AllocaNo: N);
1497 Value *AILong = IRB.CreatePointerCast(V: AI, DestTy: IntptrTy);
1498 Value *AINoTagLong = untagPointer(IRB, PtrLong: AILong);
1499 Value *Replacement = tagPointer(IRB, Ty: AI->getType(), PtrLong: AINoTagLong, Tag);
1500 std::string Name =
1501 AI->hasName() ? AI->getName().str() : "alloca." + itostr(X: N);
1502 Replacement->setName(Name + ".hwasan");
1503
1504 size_t Size = memtag::getAllocaSizeInBytes(AI: *AI);
1505 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1506
1507 AI->replaceUsesWithIf(New: Replacement, ShouldReplace: [AILong](const Use &U) {
1508 auto *User = U.getUser();
1509 return User != AILong && !isa<LifetimeIntrinsic>(Val: User);
1510 });
1511
1512 memtag::annotateDebugRecords(Info, Tag: retagMask(AllocaNo: N));
1513
1514 auto TagStarts = [&]() {
1515 for (IntrinsicInst *Start : Info.LifetimeStart) {
1516 IRB.SetInsertPoint(Start->getNextNode());
1517 tagAlloca(IRB, AI, Tag, Size);
1518 }
1519 };
1520 auto TagEnd = [&](Instruction *Node) {
1521 IRB.SetInsertPoint(Node);
1522 // When untagging, use the `AlignedSize` because we need to set the tags
1523 // for the entire alloca to original. If we used `Size` here, we would
1524 // keep the last granule tagged, and store zero in the last byte of the
1525 // last granule, due to how short granules are implemented.
1526 tagAlloca(IRB, AI, Tag: UARTag, Size: AlignedSize);
1527 };
1528 auto EraseLifetimes = [&]() {
1529 for (auto &II : Info.LifetimeStart)
1530 II->eraseFromParent();
1531 for (auto &II : Info.LifetimeEnd)
1532 II->eraseFromParent();
1533 };
1534 // Calls to functions that may return twice (e.g. setjmp) confuse the
1535 // postdominator analysis, and will leave us to keep memory tagged after
1536 // function return. Work around this by always untagging at every return
1537 // statement if return_twice functions are called.
1538 if (DetectUseAfterScope && !SInfo.CallsReturnTwice &&
1539 memtag::isSupportedLifetime(AInfo: Info, DT: &DT, LI: &LI)) {
1540 TagStarts();
1541 memtag::forAllReachableExits(DT, PDT, LI, AInfo: Info, RetVec: SInfo.RetVec, Callback: TagEnd);
1542 ORE.emit(RemarkBuilder: [&]() {
1543 return OptimizationRemark(DEBUG_TYPE, "supportedLifetime", AI);
1544 });
1545 } else if (DetectUseAfterScope && ClStrictUseAfterScope) {
1546 // SInfo.CallsReturnTwice || !isStandardLifetime
1547 ORE.emit(RemarkBuilder: [&]() {
1548 return OptimizationRemarkMissed(DEBUG_TYPE, "supportedLifetime", AI);
1549 });
1550
1551 tagAlloca(IRB, AI, Tag, Size);
1552 TagStarts();
1553 for_each(Range&: Info.LifetimeEnd, F: TagEnd);
1554 for_each(Range&: SInfo.RetVec, F: TagEnd);
1555 EraseLifetimes();
1556 } else {
1557 tagAlloca(IRB, AI, Tag, Size);
1558 for_each(Range&: SInfo.RetVec, F: TagEnd);
1559 EraseLifetimes();
1560 }
1561 memtag::alignAndPadAlloca(Info, Align: Mapping.getObjectAlignment());
1562 }
1563}
1564
1565static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE,
1566 bool Skip) {
1567 if (Skip) {
1568 ORE.emit(RemarkBuilder: [&]() {
1569 return OptimizationRemark(DEBUG_TYPE, "Skip", &F)
1570 << "Skipped: F=" << ore::NV("Function", &F);
1571 });
1572 } else {
1573 ORE.emit(RemarkBuilder: [&]() {
1574 return OptimizationRemarkMissed(DEBUG_TYPE, "Sanitize", &F)
1575 << "Sanitized: F=" << ore::NV("Function", &F);
1576 });
1577 }
1578}
1579
1580bool HWAddressSanitizer::selectiveInstrumentationShouldSkip(
1581 Function &F, FunctionAnalysisManager &FAM) const {
1582 auto SkipHot = [&]() {
1583 if (!ClHotPercentileCutoff.getNumOccurrences())
1584 return false;
1585 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1586 ProfileSummaryInfo *PSI =
1587 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
1588 if (!PSI || !PSI->hasProfileSummary()) {
1589 ++NumNoProfileSummaryFuncs;
1590 return false;
1591 }
1592 return PSI->isFunctionHotInCallGraphNthPercentile(
1593 PercentileCutoff: ClHotPercentileCutoff, F: &F, BFI&: FAM.getResult<BlockFrequencyAnalysis>(IR&: F));
1594 };
1595
1596 auto SkipRandom = [&]() {
1597 if (!ClRandomKeepRate.getNumOccurrences())
1598 return false;
1599 std::bernoulli_distribution D(ClRandomKeepRate);
1600 return !D(*Rng);
1601 };
1602
1603 bool Skip = SkipRandom() || SkipHot();
1604 emitRemark(F, ORE&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F), Skip);
1605 return Skip;
1606}
1607
1608void HWAddressSanitizer::sanitizeFunction(Function &F,
1609 FunctionAnalysisManager &FAM) {
1610 if (&F == HwasanCtorFunction)
1611 return;
1612
1613 // Do not apply any instrumentation for naked functions.
1614 if (F.hasFnAttribute(Kind: Attribute::Naked))
1615 return;
1616
1617 if (!F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1618 return;
1619
1620 if (F.empty())
1621 return;
1622
1623 if (F.isPresplitCoroutine())
1624 return;
1625
1626 NumTotalFuncs++;
1627
1628 OptimizationRemarkEmitter &ORE =
1629 FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
1630
1631 if (selectiveInstrumentationShouldSkip(F, FAM))
1632 return;
1633
1634 NumInstrumentedFuncs++;
1635
1636 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1637
1638 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1639 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1640 SmallVector<Instruction *, 8> LandingPadVec;
1641 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1642
1643 memtag::StackInfoBuilder SIB(SSI, DEBUG_TYPE);
1644 for (auto &Inst : instructions(F)) {
1645 if (InstrumentStack) {
1646 SIB.visit(ORE, Inst);
1647 }
1648
1649 if (InstrumentLandingPads && isa<LandingPadInst>(Val: Inst))
1650 LandingPadVec.push_back(Elt: &Inst);
1651
1652 getInterestingMemoryOperands(ORE, I: &Inst, TLI, Interesting&: OperandsToInstrument);
1653
1654 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst))
1655 if (!ignoreMemIntrinsic(ORE, MI))
1656 IntrinToInstrument.push_back(Elt: MI);
1657 }
1658
1659 memtag::StackInfo &SInfo = SIB.get();
1660
1661 initializeCallbacks(M&: *F.getParent());
1662
1663 if (!LandingPadVec.empty())
1664 instrumentLandingPads(LandingPadVec);
1665
1666 if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1667 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1668 // __hwasan_personality_thunk is a no-op for functions without an
1669 // instrumented stack, so we can drop it.
1670 F.setPersonalityFn(nullptr);
1671 }
1672
1673 if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1674 IntrinToInstrument.empty())
1675 return;
1676
1677 assert(!ShadowBase);
1678
1679 BasicBlock::iterator InsertPt = F.getEntryBlock().begin();
1680 IRBuilder<> EntryIRB(InsertPt);
1681 emitPrologue(IRB&: EntryIRB,
1682 /*WithFrameRecord*/ ClRecordStackHistory != none &&
1683 Mapping.withFrameRecord() &&
1684 !SInfo.AllocasToInstrument.empty());
1685
1686 if (!SInfo.AllocasToInstrument.empty()) {
1687 const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
1688 const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
1689 const LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: F);
1690 Value *StackTag = getStackBaseTag(IRB&: EntryIRB);
1691 Value *UARTag = getUARTag(IRB&: EntryIRB);
1692 instrumentStack(ORE, SInfo, StackTag, UARTag, DT, PDT, LI);
1693 }
1694
1695 // If we split the entry block, move any allocas that were originally in the
1696 // entry block back into the entry block so that they aren't treated as
1697 // dynamic allocas.
1698 if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1699 InsertPt = F.getEntryBlock().begin();
1700 for (Instruction &I :
1701 llvm::make_early_inc_range(Range&: *EntryIRB.GetInsertBlock())) {
1702 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
1703 if (isa<ConstantInt>(Val: AI->getArraySize()))
1704 I.moveBefore(BB&: F.getEntryBlock(), I: InsertPt);
1705 }
1706 }
1707
1708 DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
1709 PostDominatorTree *PDT = FAM.getCachedResult<PostDominatorTreeAnalysis>(IR&: F);
1710 LoopInfo *LI = FAM.getCachedResult<LoopAnalysis>(IR&: F);
1711 DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
1712 const DataLayout &DL = F.getDataLayout();
1713 for (auto &Operand : OperandsToInstrument)
1714 instrumentMemAccess(O&: Operand, DTU, LI, DL);
1715 DTU.flush();
1716
1717 if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1718 for (auto *Inst : IntrinToInstrument)
1719 instrumentMemIntrinsic(MI: Inst);
1720 }
1721
1722 ShadowBase = nullptr;
1723 StackBaseTag = nullptr;
1724 CachedFP = nullptr;
1725}
1726
1727void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1728 assert(!UsePageAliases);
1729 Constant *Initializer = GV->getInitializer();
1730 uint64_t SizeInBytes =
1731 M.getDataLayout().getTypeAllocSize(Ty: Initializer->getType());
1732 uint64_t NewSize = alignTo(Size: SizeInBytes, A: Mapping.getObjectAlignment());
1733 if (SizeInBytes != NewSize) {
1734 // Pad the initializer out to the next multiple of 16 bytes and add the
1735 // required short granule tag.
1736 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1737 Init.back() = Tag;
1738 Constant *Padding = ConstantDataArray::get(Context&: *C, Elts&: Init);
1739 Initializer = ConstantStruct::getAnon(V: {Initializer, Padding});
1740 }
1741
1742 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1743 GlobalValue::ExternalLinkage, Initializer,
1744 GV->getName() + ".hwasan");
1745 NewGV->copyAttributesFrom(Src: GV);
1746 NewGV->setLinkage(GlobalValue::PrivateLinkage);
1747 NewGV->copyMetadata(Src: GV, Offset: 0);
1748 NewGV->setAlignment(
1749 std::max(a: GV->getAlign().valueOrOne(), b: Mapping.getObjectAlignment()));
1750
1751 // It is invalid to ICF two globals that have different tags. In the case
1752 // where the size of the global is a multiple of the tag granularity the
1753 // contents of the globals may be the same but the tags (i.e. symbol values)
1754 // may be different, and the symbols are not considered during ICF. In the
1755 // case where the size is not a multiple of the granularity, the short granule
1756 // tags would discriminate two globals with different tags, but there would
1757 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1758 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1759 // granule tag in the last byte.
1760 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1761
1762 // Descriptor format (assuming little-endian):
1763 // bytes 0-3: relative address of global
1764 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1765 // it isn't, we create multiple descriptors)
1766 // byte 7: tag
1767 auto *DescriptorTy = StructType::get(elt1: Int32Ty, elts: Int32Ty);
1768 const uint64_t MaxDescriptorSize = 0xfffff0;
1769 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1770 DescriptorPos += MaxDescriptorSize) {
1771 auto *Descriptor =
1772 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1773 nullptr, GV->getName() + ".hwasan.descriptor");
1774 auto *GVRelPtr = ConstantExpr::getTrunc(
1775 C: ConstantExpr::getAdd(
1776 C1: ConstantExpr::getSub(
1777 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1778 C2: ConstantExpr::getPtrToInt(C: Descriptor, Ty: Int64Ty)),
1779 C2: ConstantInt::get(Ty: Int64Ty, V: DescriptorPos)),
1780 Ty: Int32Ty);
1781 uint32_t Size = std::min(a: SizeInBytes - DescriptorPos, b: MaxDescriptorSize);
1782 auto *SizeAndTag = ConstantInt::get(Ty: Int32Ty, V: Size | (uint32_t(Tag) << 24));
1783 Descriptor->setComdat(NewGV->getComdat());
1784 Descriptor->setInitializer(ConstantStruct::getAnon(V: {GVRelPtr, SizeAndTag}));
1785 Descriptor->setSection("hwasan_globals");
1786 Descriptor->setMetadata(KindID: LLVMContext::MD_associated,
1787 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: NewGV)));
1788 appendToCompilerUsed(M, Values: Descriptor);
1789 }
1790
1791 Constant *Aliasee = ConstantExpr::getIntToPtr(
1792 C: ConstantExpr::getAdd(
1793 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1794 C2: ConstantInt::get(Ty: Int64Ty, V: uint64_t(Tag) << PointerTagShift)),
1795 Ty: GV->getType());
1796 auto *Alias = GlobalAlias::create(Ty: GV->getValueType(), AddressSpace: GV->getAddressSpace(),
1797 Linkage: GV->getLinkage(), Name: "", Aliasee, Parent: &M);
1798 Alias->setVisibility(GV->getVisibility());
1799 Alias->takeName(V: GV);
1800 GV->replaceAllUsesWith(V: Alias);
1801 GV->eraseFromParent();
1802}
1803
1804void HWAddressSanitizer::instrumentGlobals() {
1805 std::vector<GlobalVariable *> Globals;
1806 for (GlobalVariable &GV : M.globals()) {
1807 if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1808 continue;
1809
1810 if (GV.isDeclarationForLinker() || GV.getName().starts_with(Prefix: "llvm.") ||
1811 GV.isThreadLocal())
1812 continue;
1813
1814 // Common symbols can't have aliases point to them, so they can't be tagged.
1815 if (GV.hasCommonLinkage())
1816 continue;
1817
1818 if (ClAllGlobals) {
1819 // Avoid instrumenting intrinsic global variables.
1820 if (GV.getSection() == "llvm.metadata")
1821 continue;
1822 } else {
1823 // Globals with custom sections may be used in __start_/__stop_
1824 // enumeration, which would be broken both by adding tags and potentially
1825 // by the extra padding/alignment that we insert.
1826 if (GV.hasSection())
1827 continue;
1828 }
1829
1830 Globals.push_back(x: &GV);
1831 }
1832
1833 MD5 Hasher;
1834 Hasher.update(Str: M.getSourceFileName());
1835 MD5::MD5Result Hash;
1836 Hasher.final(Result&: Hash);
1837 uint8_t Tag = Hash[0];
1838
1839 assert(TagMaskByte >= 16);
1840
1841 for (GlobalVariable *GV : Globals) {
1842 // Don't allow globals to be tagged with something that looks like a
1843 // short-granule tag, otherwise we lose inter-granule overflow detection, as
1844 // the fast path shadow-vs-address check succeeds.
1845 if (Tag < 16 || Tag > TagMaskByte)
1846 Tag = 16;
1847 instrumentGlobal(GV, Tag: Tag++);
1848 }
1849}
1850
1851void HWAddressSanitizer::instrumentPersonalityFunctions() {
1852 // We need to untag stack frames as we unwind past them. That is the job of
1853 // the personality function wrapper, which either wraps an existing
1854 // personality function or acts as a personality function on its own. Each
1855 // function that has a personality function or that can be unwound past has
1856 // its personality function changed to a thunk that calls the personality
1857 // function wrapper in the runtime.
1858 MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1859 for (Function &F : M) {
1860 if (F.isDeclaration() || !F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1861 continue;
1862
1863 if (F.hasPersonalityFn()) {
1864 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(x: &F);
1865 } else if (!F.hasFnAttribute(Kind: Attribute::NoUnwind)) {
1866 PersonalityFns[nullptr].push_back(x: &F);
1867 }
1868 }
1869
1870 if (PersonalityFns.empty())
1871 return;
1872
1873 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1874 Name: "__hwasan_personality_wrapper", RetTy: Int32Ty, Args: Int32Ty, Args: Int32Ty, Args: Int64Ty, Args: PtrTy,
1875 Args: PtrTy, Args: PtrTy, Args: PtrTy, Args: PtrTy);
1876 FunctionCallee UnwindGetGR = M.getOrInsertFunction(Name: "_Unwind_GetGR", RetTy: VoidTy);
1877 FunctionCallee UnwindGetCFA = M.getOrInsertFunction(Name: "_Unwind_GetCFA", RetTy: VoidTy);
1878
1879 for (auto &P : PersonalityFns) {
1880 std::string ThunkName = kHwasanPersonalityThunkName;
1881 if (P.first)
1882 ThunkName += ("." + P.first->getName()).str();
1883 FunctionType *ThunkFnTy = FunctionType::get(
1884 Result: Int32Ty, Params: {Int32Ty, Int32Ty, Int64Ty, PtrTy, PtrTy}, isVarArg: false);
1885 bool IsLocal = P.first && (!isa<GlobalValue>(Val: P.first) ||
1886 cast<GlobalValue>(Val: P.first)->hasLocalLinkage());
1887 auto *ThunkFn = Function::Create(Ty: ThunkFnTy,
1888 Linkage: IsLocal ? GlobalValue::InternalLinkage
1889 : GlobalValue::LinkOnceODRLinkage,
1890 N: ThunkName, M: &M);
1891 // TODO: think about other attributes as well.
1892 if (any_of(Range&: P.second, P: [](const Function *F) {
1893 return F->hasFnAttribute(Kind: "branch-target-enforcement");
1894 })) {
1895 ThunkFn->addFnAttr(Kind: "branch-target-enforcement");
1896 }
1897 if (!IsLocal) {
1898 ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1899 ThunkFn->setComdat(M.getOrInsertComdat(Name: ThunkName));
1900 }
1901
1902 auto *BB = BasicBlock::Create(Context&: *C, Name: "entry", Parent: ThunkFn);
1903 IRBuilder<> IRB(BB);
1904 CallInst *WrapperCall = IRB.CreateCall(
1905 Callee: HwasanPersonalityWrapper,
1906 Args: {ThunkFn->getArg(i: 0), ThunkFn->getArg(i: 1), ThunkFn->getArg(i: 2),
1907 ThunkFn->getArg(i: 3), ThunkFn->getArg(i: 4),
1908 P.first ? P.first : Constant::getNullValue(Ty: PtrTy),
1909 UnwindGetGR.getCallee(), UnwindGetCFA.getCallee()});
1910 WrapperCall->setTailCall();
1911 IRB.CreateRet(V: WrapperCall);
1912
1913 for (Function *F : P.second)
1914 F->setPersonalityFn(ThunkFn);
1915 }
1916}
1917
1918void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1919 bool InstrumentWithCalls,
1920 bool CompileKernel) {
1921 // Start with defaults.
1922 Scale = kDefaultShadowScale;
1923 Kind = OffsetKind::kTls;
1924 WithFrameRecord = true;
1925
1926 // Tune for the target.
1927 if (TargetTriple.isOSFuchsia()) {
1928 // Fuchsia is always PIE, which means that the beginning of the address
1929 // space is always available.
1930 Kind = OffsetKind::kGlobal;
1931 } else if (CompileKernel || InstrumentWithCalls) {
1932 SetFixed(0);
1933 WithFrameRecord = false;
1934 }
1935
1936 WithFrameRecord = optOr(Opt&: ClFrameRecords, Other: WithFrameRecord);
1937
1938 // Apply the last of ClMappingOffset and ClMappingOffsetDynamic.
1939 Kind = optOr(Opt&: ClMappingOffsetDynamic, Other: Kind);
1940 if (ClMappingOffset.getNumOccurrences() > 0 &&
1941 !(ClMappingOffsetDynamic.getNumOccurrences() > 0 &&
1942 ClMappingOffsetDynamic.getPosition() > ClMappingOffset.getPosition())) {
1943 SetFixed(ClMappingOffset);
1944 }
1945}
1946