1//===- HWAddressSanitizer.cpp - memory access error detector --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of HWAddressSanitizer, an address basic correctness
11/// checker based on tagged addressing.
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15#include "InstrumentationOptions.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/BlockFrequencyInfo.h"
23#include "llvm/Analysis/DomTreeUpdater.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/OptimizationRemarkEmitter.h"
26#include "llvm/Analysis/PostDominators.h"
27#include "llvm/Analysis/ProfileSummaryInfo.h"
28#include "llvm/Analysis/StackSafetyAnalysis.h"
29#include "llvm/Analysis/TargetLibraryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/BinaryFormat/ELF.h"
33#include "llvm/IR/Attributes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/DerivedTypes.h"
39#include "llvm/IR/Dominators.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/LLVMContext.h"
49#include "llvm/IR/MDBuilder.h"
50#include "llvm/IR/Module.h"
51#include "llvm/IR/Type.h"
52#include "llvm/IR/Value.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/MD5.h"
58#include "llvm/Support/RandomNumberGenerator.h"
59#include "llvm/Support/raw_ostream.h"
60#include "llvm/TargetParser/Triple.h"
61#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
62#include "llvm/Transforms/Utils/BasicBlockUtils.h"
63#include "llvm/Transforms/Utils/Instrumentation.h"
64#include "llvm/Transforms/Utils/Local.h"
65#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
66#include "llvm/Transforms/Utils/ModuleUtils.h"
67#include "llvm/Transforms/Utils/PromoteMemToReg.h"
68#include <optional>
69#include <random>
70
71using namespace llvm;
72
73#define DEBUG_TYPE "hwasan"
74
75const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
76const char kHwasanNoteName[] = "hwasan.note";
77const char kHwasanInitName[] = "__hwasan_init";
78const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
79
80const char kHwasanShadowMemoryDynamicAddress[] =
81 "__hwasan_shadow_memory_dynamic_address";
82
83// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
84static const size_t kNumberOfAccessSizes = 5;
85
86static const size_t kDefaultShadowScale = 4;
87
88static const unsigned kShadowBaseAlignment = 32;
89
90namespace {
91enum class OffsetKind {
92 kFixed = 0,
93 kGlobal,
94 kIfunc,
95 kTls,
96};
97}
98
99// These flags allow to change the shadow mapping and control how shadow memory
100// is accessed. The shadow mapping looks like:
101// Shadow = (Mem >> scale) + offset
102
103static cl::opt<uint64_t>
104 ClMappingOffset("hwasan-mapping-offset",
105 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
106 cl::Hidden);
107
108static cl::opt<OffsetKind> ClMappingOffsetDynamic(
109 "hwasan-mapping-offset-dynamic",
110 cl::desc("HWASan shadow mapping dynamic offset location"), cl::Hidden,
111 cl::values(clEnumValN(OffsetKind::kGlobal, "global", "Use global"),
112 clEnumValN(OffsetKind::kIfunc, "ifunc", "Use ifunc global"),
113 clEnumValN(OffsetKind::kTls, "tls", "Use TLS")));
114
115STATISTIC(NumTotalFuncs, "Number of total funcs");
116STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs");
117STATISTIC(NumNoProfileSummaryFuncs, "Number of funcs without PS");
118
119namespace {
120
121bool shouldUsePageAliases(const InstrumentationOptions &Opts,
122 const Triple &TargetTriple) {
123 return Opts.hwasan_experimental_use_page_aliases &&
124 TargetTriple.getArch() == Triple::x86_64;
125}
126
127bool shouldInstrumentStack(const InstrumentationOptions &Opts,
128 const Triple &TargetTriple) {
129 return !shouldUsePageAliases(Opts, TargetTriple) &&
130 Opts.hwasan_instrument_stack;
131}
132
133bool shouldInstrumentWithCalls(const InstrumentationOptions &Opts,
134 const Triple &TargetTriple) {
135 return valueOr(X: Opts.hwasan_instrument_with_calls,
136 Default: TargetTriple.getArch() == Triple::x86_64);
137}
138
139bool mightUseStackSafetyAnalysis(const InstrumentationOptions &Opts,
140 bool DisableOptimization) {
141 return valueOr(X: Opts.hwasan_use_stack_safety, Default: !DisableOptimization);
142}
143
144bool shouldUseStackSafetyAnalysis(const InstrumentationOptions &Opts,
145 const Triple &TargetTriple,
146 bool DisableOptimization) {
147 return shouldInstrumentStack(Opts, TargetTriple) &&
148 mightUseStackSafetyAnalysis(Opts, DisableOptimization);
149}
150
151bool shouldDetectUseAfterScope(const InstrumentationOptions &Opts,
152 const Triple &TargetTriple) {
153 return Opts.hwasan_use_after_scope &&
154 shouldInstrumentStack(Opts, TargetTriple);
155}
156
157/// An instrumentation pass implementing detection of addressability bugs
158/// using tagged pointers.
159class HWAddressSanitizer {
160public:
161 HWAddressSanitizer(const InstrumentationOptions &Opts, Module &M,
162 bool CompileKernel, bool Recover,
163 const StackSafetyGlobalInfo *SSI)
164 : Opts(Opts), M(M), SSI(SSI) {
165 this->Recover = valueOr(X: Opts.hwasan_recover, Default: Recover);
166 this->CompileKernel = valueOr(X: Opts.hwasan_kernel, Default: CompileKernel);
167 this->Rng = Opts.hwasan_random_rate ? M.createRNG(DEBUG_TYPE) : nullptr;
168
169 initializeModule();
170 }
171
172 void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
173
174private:
175 struct ShadowTagCheckInfo {
176 Instruction *TagMismatchTerm = nullptr;
177 Value *PtrLong = nullptr;
178 Value *AddrLong = nullptr;
179 Value *PtrTag = nullptr;
180 Value *MemTag = nullptr;
181 };
182
183 bool selectiveInstrumentationShouldSkip(Function &F,
184 FunctionAnalysisManager &FAM) const;
185 void initializeModule();
186 void createHwasanCtorComdat();
187 void createHwasanNote();
188
189 void initializeCallbacks(Module &M);
190
191 Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
192
193 Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
194 Value *getShadowNonTls(IRBuilder<> &IRB);
195
196 void untagPointerOperand(Instruction *I, Value *Addr);
197 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
198
199 int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
200 ShadowTagCheckInfo insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
201 DomTreeUpdater &DTU, LoopInfo *LI);
202 void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
203 unsigned AccessSizeIndex,
204 Instruction *InsertBefore,
205 DomTreeUpdater &DTU, LoopInfo *LI);
206 void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
207 unsigned AccessSizeIndex,
208 Instruction *InsertBefore, DomTreeUpdater &DTU,
209 LoopInfo *LI);
210 bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI);
211 void instrumentMemIntrinsic(MemIntrinsic *MI);
212 bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
213 LoopInfo *LI, const DataLayout &DL);
214 bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr);
215 bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst,
216 Value *Ptr);
217
218 void getInterestingMemoryOperands(
219 OptimizationRemarkEmitter &ORE, Instruction *I,
220 const TargetLibraryInfo &TLI,
221 SmallVectorImpl<InterestingMemoryOperand> &Interesting);
222
223 void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
224 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
225 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
226 void instrumentStack(OptimizationRemarkEmitter &ORE, memtag::StackInfo &Info,
227 Value *StackTag, Value *UARTag, const DominatorTree &DT,
228 const PostDominatorTree &PDT, const LoopInfo &LI);
229 void instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
230 Value *getNextTagWithCall(IRBuilder<> &IRB);
231 Value *getStackBaseTag(IRBuilder<> &IRB);
232 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
233 Value *getUARTag(IRBuilder<> &IRB);
234
235 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB);
236 Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
237 unsigned retagMask(unsigned AllocaNo);
238
239 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
240
241 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
242 void instrumentGlobals();
243
244 Value *getCachedFP(IRBuilder<> &IRB);
245 Value *getFrameRecordInfo(IRBuilder<> &IRB);
246
247 void instrumentPersonalityFunctions();
248
249 const InstrumentationOptions &Opts;
250 LLVMContext *C;
251 Module &M;
252 const StackSafetyGlobalInfo *SSI;
253 Triple TargetTriple;
254 std::unique_ptr<RandomNumberGenerator> Rng;
255
256 /// This struct defines the shadow mapping using the rule:
257 /// If `kFixed`, then
258 /// shadow = (mem >> Scale) + Offset.
259 /// If `kGlobal`, then
260 /// extern char* __hwasan_shadow_memory_dynamic_address;
261 /// shadow = (mem >> Scale) + __hwasan_shadow_memory_dynamic_address
262 /// If `kIfunc`, then
263 /// extern char __hwasan_shadow[];
264 /// shadow = (mem >> Scale) + &__hwasan_shadow
265 /// If `kTls`, then
266 /// extern char *__hwasan_tls;
267 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
268 ///
269 /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
270 /// ring buffer for storing stack allocations on targets that support it.
271 class ShadowMapping {
272 OffsetKind Kind;
273 uint64_t Offset;
274 uint8_t Scale;
275 bool WithFrameRecord;
276
277 void SetFixed(uint64_t O) {
278 Kind = OffsetKind::kFixed;
279 Offset = O;
280 }
281
282 public:
283 void init(const InstrumentationOptions &Opts, Triple &TargetTriple,
284 bool InstrumentWithCalls, bool CompileKernel);
285 Align getObjectAlignment() const { return Align::fromLog2(Log2Value: Scale); }
286 bool isInGlobal() const { return Kind == OffsetKind::kGlobal; }
287 bool isInIfunc() const { return Kind == OffsetKind::kIfunc; }
288 bool isInTls() const { return Kind == OffsetKind::kTls; }
289 bool isFixed() const { return Kind == OffsetKind::kFixed; }
290 uint8_t scale() const { return Scale; };
291 uint64_t offset() const {
292 assert(isFixed());
293 return Offset;
294 };
295 bool withFrameRecord() const { return WithFrameRecord; };
296 };
297
298 ShadowMapping Mapping;
299
300 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
301 Type *IntptrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
302 PointerType *PtrTy = PointerType::getUnqual(C&: M.getContext());
303 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
304 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
305 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
306
307 bool CompileKernel;
308 bool Recover;
309 bool OutlinedChecks;
310 bool InlineFastPath;
311 bool UseShortGranules;
312 bool InstrumentLandingPads;
313 bool InstrumentWithCalls;
314 bool InstrumentStack;
315 bool InstrumentGlobals;
316 bool DetectUseAfterScope;
317 bool UsePageAliases;
318 bool UseMatchAllCallback;
319
320 std::optional<uint8_t> MatchAllTag;
321
322 unsigned PointerTagShift;
323 uint64_t TagMaskByte;
324
325 Function *HwasanCtorFunction;
326
327 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
328 FunctionCallee HwasanMemoryAccessCallbackSized[2];
329
330 FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
331 FunctionCallee HwasanHandleVfork;
332
333 FunctionCallee HwasanTagMemoryFunc;
334 FunctionCallee HwasanGenerateTagFunc;
335 FunctionCallee HwasanRecordFrameRecordFunc;
336
337 Constant *ShadowGlobal;
338
339 Value *ShadowBase = nullptr;
340 Value *StackBaseTag = nullptr;
341 Value *CachedFP = nullptr;
342 GlobalValue *ThreadPtrGlobal = nullptr;
343};
344
345} // end anonymous namespace
346
347PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
348 ModuleAnalysisManager &MAM) {
349 // Return early if nosanitize_hwaddress module flag is present for the module.
350 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_hwaddress"))
351 return PreservedAnalyses::all();
352 const StackSafetyGlobalInfo *SSI = nullptr;
353 const Triple &TargetTriple = M.getTargetTriple();
354 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
355 if (shouldUseStackSafetyAnalysis(Opts, TargetTriple,
356 DisableOptimization: Options.DisableOptimization))
357 SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M);
358
359 HWAddressSanitizer HWASan(Opts, M, Options.CompileKernel, Options.Recover,
360 SSI);
361 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
362 for (Function &F : M)
363 HWASan.sanitizeFunction(F, FAM);
364
365 PreservedAnalyses PA = PreservedAnalyses::none();
366 // DominatorTreeAnalysis, PostDominatorTreeAnalysis, and LoopAnalysis
367 // are incrementally updated throughout this pass whenever
368 // SplitBlockAndInsertIfThen is called.
369 PA.preserve<DominatorTreeAnalysis>();
370 PA.preserve<PostDominatorTreeAnalysis>();
371 PA.preserve<LoopAnalysis>();
372 // GlobalsAA is considered stateless and does not get invalidated unless
373 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
374 // make changes that require GlobalsAA to be invalidated.
375 PA.abandon<GlobalsAA>();
376 return PA;
377}
378void HWAddressSanitizerPass::printPipeline(
379 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
380 static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
381 OS, MapClassName2PassName);
382 OS << '<';
383 if (Options.CompileKernel)
384 OS << "kernel;";
385 if (Options.Recover)
386 OS << "recover";
387 OS << '>';
388}
389
390void HWAddressSanitizer::createHwasanNote() {
391 // Create a note that contains pointers to the list of global
392 // descriptors. Adding a note to the output file will cause the linker to
393 // create a PT_NOTE program header pointing to the note that we can use to
394 // find the descriptor list starting from the program headers. A function
395 // provided by the runtime initializes the shadow memory for the globals by
396 // accessing the descriptor list via the note. The dynamic loader needs to
397 // call this function whenever a library is loaded.
398 //
399 // The reason why we use a note for this instead of a more conventional
400 // approach of having a global constructor pass a descriptor list pointer to
401 // the runtime is because of an order of initialization problem. With
402 // constructors we can encounter the following problematic scenario:
403 //
404 // 1) library A depends on library B and also interposes one of B's symbols
405 // 2) B's constructors are called before A's (as required for correctness)
406 // 3) during construction, B accesses one of its "own" globals (actually
407 // interposed by A) and triggers a HWASAN failure due to the initialization
408 // for A not having happened yet
409 //
410 // Even without interposition it is possible to run into similar situations in
411 // cases where two libraries mutually depend on each other.
412 //
413 // We only need one note per binary, so put everything for the note in a
414 // comdat. This needs to be a comdat with an .init_array section to prevent
415 // newer versions of lld from discarding the note.
416 //
417 // Create the note even if we aren't instrumenting globals. This ensures that
418 // binaries linked from object files with both instrumented and
419 // non-instrumented globals will end up with a note, even if a comdat from an
420 // object file with non-instrumented globals is selected. The note is harmless
421 // if the runtime doesn't support it, since it will just be ignored.
422 Comdat *NoteComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
423
424 Type *Int8Arr0Ty = ArrayType::get(ElementType: Int8Ty, NumElements: 0);
425 auto *Start =
426 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
427 nullptr, "__start_hwasan_globals");
428 Start->setVisibility(GlobalValue::HiddenVisibility);
429 auto *Stop =
430 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
431 nullptr, "__stop_hwasan_globals");
432 Stop->setVisibility(GlobalValue::HiddenVisibility);
433
434 // Null-terminated so actually 8 bytes, which are required in order to align
435 // the note properly.
436 auto *Name = ConstantDataArray::get(Context&: *C, Elts: "LLVM\0\0\0");
437
438 auto *NoteTy = StructType::get(elt1: Int32Ty, elts: Int32Ty, elts: Int32Ty, elts: Name->getType(),
439 elts: Int32Ty, elts: Int32Ty);
440 auto *Note =
441 new GlobalVariable(M, NoteTy, /*isConstant=*/true,
442 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
443 Note->setSection(".note.hwasan.globals");
444 Note->setComdat(NoteComdat);
445 Note->setAlignment(Align(4));
446
447 // The pointers in the note need to be relative so that the note ends up being
448 // placed in rodata, which is the standard location for notes.
449 auto CreateRelPtr = [&](Constant *Ptr) {
450 return ConstantExpr::getTrunc(
451 C: ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: Ptr, Ty: Int64Ty),
452 C2: ConstantExpr::getPtrToInt(C: Note, Ty: Int64Ty)),
453 Ty: Int32Ty);
454 };
455 Note->setInitializer(ConstantStruct::getAnon(
456 V: {ConstantInt::get(Ty: Int32Ty, V: 8), // n_namesz
457 ConstantInt::get(Ty: Int32Ty, V: 8), // n_descsz
458 ConstantInt::get(Ty: Int32Ty, V: ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
459 Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
460 appendToCompilerUsed(M, Values: Note);
461
462 // Create a zero-length global in hwasan_globals so that the linker will
463 // always create start and stop symbols.
464 auto *Dummy = new GlobalVariable(
465 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
466 Constant::getNullValue(Ty: Int8Arr0Ty), "hwasan.dummy.global");
467 Dummy->setSection("hwasan_globals");
468 Dummy->setComdat(NoteComdat);
469 Dummy->setMetadata(KindID: LLVMContext::MD_associated,
470 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: Note)));
471 appendToCompilerUsed(M, Values: Dummy);
472}
473
474void HWAddressSanitizer::createHwasanCtorComdat() {
475 std::tie(args&: HwasanCtorFunction, args: std::ignore) =
476 getOrCreateSanitizerCtorAndInitFunctions(
477 M, CtorName: kHwasanModuleCtorName, InitName: kHwasanInitName,
478 /*InitArgTypes=*/{},
479 /*InitArgs=*/{},
480 // This callback is invoked when the functions are created the first
481 // time. Hook them into the global ctors list in that case:
482 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
483 Comdat *CtorComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
484 Ctor->setComdat(CtorComdat);
485 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
486 });
487
488 // Do not create .note.hwasan.globals for static binaries, as it is only
489 // needed for instrumenting globals from dynamic libraries. In static
490 // binaries, the global variables section can be accessed directly via the
491 // __start_hwasan_globals and __stop_hwasan_globals symbols inserted by the
492 // linker.
493 if (!Opts.hwasan_static_linking)
494 createHwasanNote();
495}
496
497/// Module-level initialization.
498///
499/// inserts a call to __hwasan_init to the module's constructor list.
500void HWAddressSanitizer::initializeModule() {
501 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
502 TargetTriple = M.getTargetTriple();
503
504 // HWASan may do short granule checks on function arguments read from the
505 // argument memory (last byte of the granule), which invalidates writeonly.
506 for (Function &F : M.functions())
507 removeASanIncompatibleFnAttributes(F, /*ReadsArgMem=*/true);
508
509 // x86_64 currently has two modes:
510 // - Intel LAM (default)
511 // - pointer aliasing (heap only)
512 bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
513 UsePageAliases = shouldUsePageAliases(Opts, TargetTriple);
514 InstrumentWithCalls = shouldInstrumentWithCalls(Opts, TargetTriple);
515 InstrumentStack = shouldInstrumentStack(Opts, TargetTriple);
516 DetectUseAfterScope = shouldDetectUseAfterScope(Opts, TargetTriple);
517 PointerTagShift = IsX86_64 ? 57 : 56;
518 TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
519 if (Opts.hwasan_tag_bits) {
520 if (TagMaskByte < 4)
521 reportFatalUsageError(
522 reason: "need more than 4 bits of tag to have non-short-granule tags");
523 TagMaskByte &= (1ULL << Opts.hwasan_tag_bits) - 1;
524 }
525
526 Mapping.init(Opts, TargetTriple, InstrumentWithCalls, CompileKernel);
527
528 C = &(M.getContext());
529
530 HwasanCtorFunction = nullptr;
531
532 // Older versions of Android do not have the required runtime support for
533 // short granules, global or personality function instrumentation. On other
534 // platforms we currently require using the latest version of the runtime.
535 bool NewRuntime =
536 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(Major: 30);
537
538 UseShortGranules = valueOr(X: Opts.hwasan_use_short_granules, Default: NewRuntime);
539 OutlinedChecks = (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
540 TargetTriple.isOSBinFormatELF() &&
541 !valueOr(X: Opts.hwasan_inline_all_checks, Default: Recover);
542
543 // These platforms may prefer less inlining to reduce binary size.
544 InlineFastPath =
545 valueOr(X: Opts.hwasan_inline_fast_path_checks,
546 Default: !(TargetTriple.isAndroid() || TargetTriple.isOSFuchsia()));
547
548 if (Opts.hwasan_match_all_tag) {
549 if (*Opts.hwasan_match_all_tag != -1) {
550 MatchAllTag = *Opts.hwasan_match_all_tag & 0xFF;
551 }
552 } else if (CompileKernel) {
553 MatchAllTag = 0xFF;
554 }
555 UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
556
557 // If we don't have personality function support, fall back to landing pads.
558 InstrumentLandingPads =
559 valueOr(X: Opts.hwasan_instrument_landing_pads, Default: !NewRuntime);
560
561 InstrumentGlobals = !CompileKernel && !UsePageAliases &&
562 valueOr(X: Opts.hwasan_globals, Default: NewRuntime);
563
564 if (!CompileKernel) {
565 if (InstrumentGlobals)
566 instrumentGlobals();
567
568 createHwasanCtorComdat();
569
570 bool InstrumentPersonalityFunctions =
571 valueOr(X: Opts.hwasan_instrument_personality_functions, Default: NewRuntime);
572 if (InstrumentPersonalityFunctions)
573 instrumentPersonalityFunctions();
574 }
575
576 if (!TargetTriple.isAndroid()) {
577 ThreadPtrGlobal = M.getOrInsertGlobal(Name: "__hwasan_tls", Ty: IntptrTy, CreateGlobalCallback: [&] {
578 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
579 GlobalValue::ExternalLinkage, nullptr,
580 "__hwasan_tls", nullptr,
581 GlobalVariable::InitialExecTLSModel);
582 appendToCompilerUsed(M, Values: GV);
583 return GV;
584 });
585 }
586}
587
588void HWAddressSanitizer::initializeCallbacks(Module &M) {
589 const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
590 FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
591 *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
592 *HwasanMemsetFnTy;
593 if (UseMatchAllCallback) {
594 HwasanMemoryAccessCallbackSizedFnTy =
595 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy, Int8Ty}, isVarArg: false);
596 HwasanMemoryAccessCallbackFnTy =
597 FunctionType::get(Result: VoidTy, Params: {IntptrTy, Int8Ty}, isVarArg: false);
598 HwasanMemTransferFnTy =
599 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy, Int8Ty}, isVarArg: false);
600 HwasanMemsetFnTy =
601 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy, Int8Ty}, isVarArg: false);
602 } else {
603 HwasanMemoryAccessCallbackSizedFnTy =
604 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy}, isVarArg: false);
605 HwasanMemoryAccessCallbackFnTy =
606 FunctionType::get(Result: VoidTy, Params: {IntptrTy}, isVarArg: false);
607 HwasanMemTransferFnTy =
608 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy}, isVarArg: false);
609 HwasanMemsetFnTy =
610 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy}, isVarArg: false);
611 }
612
613 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
614 const std::string TypeStr = AccessIsWrite ? "store" : "load";
615 const std::string EndingStr = Recover ? "_noabort" : "";
616
617 HwasanMemoryAccessCallbackSized[AccessIsWrite] =
618 M.getOrInsertFunction(Name: (Opts.hwasan_memory_access_callback_prefix +
619 TypeStr + "N" + MatchAllStr + EndingStr)
620 .str(),
621 T: HwasanMemoryAccessCallbackSizedFnTy);
622
623 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
624 AccessSizeIndex++) {
625 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
626 M.getOrInsertFunction(Name: (Opts.hwasan_memory_access_callback_prefix +
627 TypeStr + itostr(X: 1ULL << AccessSizeIndex) +
628 MatchAllStr + EndingStr)
629 .str(),
630 T: HwasanMemoryAccessCallbackFnTy);
631 }
632 }
633
634 const std::string MemIntrinCallbackPrefix =
635 (CompileKernel && !Opts.hwasan_kernel_mem_intrinsic_prefix)
636 ? std::string("")
637 : Opts.hwasan_memory_access_callback_prefix.str();
638
639 HwasanMemmove = M.getOrInsertFunction(
640 Name: MemIntrinCallbackPrefix + "memmove" + MatchAllStr, T: HwasanMemTransferFnTy);
641 HwasanMemcpy = M.getOrInsertFunction(
642 Name: MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, T: HwasanMemTransferFnTy);
643 HwasanMemset = M.getOrInsertFunction(
644 Name: MemIntrinCallbackPrefix + "memset" + MatchAllStr, T: HwasanMemsetFnTy);
645
646 HwasanTagMemoryFunc = M.getOrInsertFunction(Name: "__hwasan_tag_memory", RetTy: VoidTy,
647 Args: PtrTy, Args: Int8Ty, Args: IntptrTy);
648 HwasanGenerateTagFunc =
649 M.getOrInsertFunction(Name: "__hwasan_generate_tag", RetTy: Int8Ty);
650
651 HwasanRecordFrameRecordFunc =
652 M.getOrInsertFunction(Name: "__hwasan_add_frame_record", RetTy: VoidTy, Args: Int64Ty);
653
654 ShadowGlobal =
655 M.getOrInsertGlobal(Name: "__hwasan_shadow", Ty: ArrayType::get(ElementType: Int8Ty, NumElements: 0));
656
657 HwasanHandleVfork =
658 M.getOrInsertFunction(Name: "__hwasan_handle_vfork", RetTy: VoidTy, Args: IntptrTy);
659}
660
661Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
662 // An empty inline asm with input reg == output reg.
663 // An opaque no-op cast, basically.
664 // This prevents code bloat as a result of rematerializing trivial definitions
665 // such as constants or global addresses at every load and store.
666 InlineAsm *Asm =
667 InlineAsm::get(Ty: FunctionType::get(Result: PtrTy, Params: {Val->getType()}, isVarArg: false),
668 AsmString: StringRef(""), Constraints: StringRef("=r,0"),
669 /*hasSideEffects=*/false);
670 return IRB.CreateCall(Callee: Asm, Args: {Val}, Name: ".hwasan.shadow");
671}
672
673Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
674 return getOpaqueNoopCast(IRB, Val: ShadowGlobal);
675}
676
677Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
678 if (Mapping.isFixed()) {
679 return getOpaqueNoopCast(
680 IRB, Val: ConstantExpr::getIntToPtr(
681 C: ConstantInt::get(Ty: IntptrTy, V: Mapping.offset()), Ty: PtrTy));
682 }
683
684 if (Mapping.isInIfunc())
685 return getDynamicShadowIfunc(IRB);
686
687 Value *GlobalDynamicAddress = IRB.getModule()->getOrInsertGlobal(
688 Name: kHwasanShadowMemoryDynamicAddress, Ty: PtrTy);
689 return IRB.CreateLoad(Ty: PtrTy, Ptr: GlobalDynamicAddress);
690}
691
692bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst,
693 Value *Ptr) {
694 // Do not instrument accesses from different address spaces; we cannot deal
695 // with them.
696 Type *PtrTy = cast<PointerType>(Val: Ptr->getType()->getScalarType());
697 if (PtrTy->getPointerAddressSpace() != 0)
698 return true;
699
700 // Ignore swifterror addresses.
701 // swifterror memory addresses are mem2reg promoted by instruction
702 // selection. As such they cannot have regular uses like an instrumentation
703 // function and it makes no sense to track them as memory.
704 if (Ptr->isSwiftError())
705 return true;
706
707 if (findAllocaForValue(V: Ptr)) {
708 if (!InstrumentStack)
709 return true;
710 if (SSI && SSI->stackAccessIsSafe(I: *Inst))
711 return true;
712 }
713
714 if (isa<GlobalVariable>(Val: getUnderlyingObject(V: Ptr))) {
715 if (!InstrumentGlobals)
716 return true;
717 // TODO: Optimize inbound global accesses, like Asan `instrumentMop`.
718 }
719
720 return false;
721}
722
723bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE,
724 Instruction *Inst, Value *Ptr) {
725 bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr);
726 if (Ignored) {
727 ORE.emit(
728 RemarkBuilder: [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); });
729 } else {
730 ORE.emit(RemarkBuilder: [&]() {
731 return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst);
732 });
733 }
734 return Ignored;
735}
736
737void HWAddressSanitizer::getInterestingMemoryOperands(
738 OptimizationRemarkEmitter &ORE, Instruction *I,
739 const TargetLibraryInfo &TLI,
740 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
741 // Skip memory accesses inserted by another instrumentation.
742 if (I->hasMetadata(KindID: LLVMContext::MD_nosanitize))
743 return;
744
745 // Do not instrument the load fetching the dynamic shadow address.
746 if (ShadowBase == I)
747 return;
748
749 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
750 if (!Opts.hwasan_instrument_reads ||
751 ignoreAccess(ORE, Inst: I, Ptr: LI->getPointerOperand()))
752 return;
753 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
754 Args: LI->getType(), Args: LI->getAlign());
755 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
756 if (!Opts.hwasan_instrument_writes ||
757 ignoreAccess(ORE, Inst: I, Ptr: SI->getPointerOperand()))
758 return;
759 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
760 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
761 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
762 if (!Opts.hwasan_instrument_atomics ||
763 ignoreAccess(ORE, Inst: I, Ptr: RMW->getPointerOperand()))
764 return;
765 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
766 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
767 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
768 if (!Opts.hwasan_instrument_atomics ||
769 ignoreAccess(ORE, Inst: I, Ptr: XCHG->getPointerOperand()))
770 return;
771 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
772 Args: XCHG->getCompareOperand()->getType(),
773 Args: std::nullopt);
774 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
775 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
776 if (!Opts.hwasan_instrument_byval || !CI->isByValArgument(ArgNo) ||
777 ignoreAccess(ORE, Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
778 continue;
779 Type *Ty = CI->getParamByValType(ArgNo);
780 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
781 }
782 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
783 }
784}
785
786static unsigned getPointerOperandIndex(Instruction *I) {
787 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I))
788 return LI->getPointerOperandIndex();
789 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I))
790 return SI->getPointerOperandIndex();
791 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I))
792 return RMW->getPointerOperandIndex();
793 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I))
794 return XCHG->getPointerOperandIndex();
795 report_fatal_error(reason: "Unexpected instruction");
796 return -1;
797}
798
799static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
800 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
801 assert(Res < kNumberOfAccessSizes);
802 return Res;
803}
804
805void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
806 if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
807 TargetTriple.isRISCV64())
808 return;
809
810 IRBuilder<> IRB(I);
811 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
812 Value *UntaggedPtr =
813 IRB.CreateIntToPtr(V: untagPointer(IRB, PtrLong: AddrLong), DestTy: Addr->getType());
814 I->setOperand(i: getPointerOperandIndex(I), Val: UntaggedPtr);
815}
816
817Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
818 // Mem >> Scale
819 Value *Shadow = IRB.CreateLShr(LHS: Mem, RHS: Mapping.scale());
820 if (Mapping.isFixed() && Mapping.offset() == 0)
821 return IRB.CreateIntToPtr(V: Shadow, DestTy: PtrTy);
822 // (Mem >> Scale) + Offset
823 return IRB.CreatePtrAdd(Ptr: ShadowBase, Offset: Shadow);
824}
825
826int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
827 unsigned AccessSizeIndex) {
828 return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
829 (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
830 (MatchAllTag.value_or(u: 0) << HWASanAccessInfo::MatchAllShift) |
831 (Recover << HWASanAccessInfo::RecoverShift) |
832 (IsWrite << HWASanAccessInfo::IsWriteShift) |
833 (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
834}
835
836HWAddressSanitizer::ShadowTagCheckInfo
837HWAddressSanitizer::insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
838 DomTreeUpdater &DTU, LoopInfo *LI) {
839 ShadowTagCheckInfo R;
840
841 IRBuilder<> IRB(InsertBefore);
842
843 R.PtrLong = IRB.CreatePointerCast(V: Ptr, DestTy: IntptrTy);
844 R.PtrTag =
845 IRB.CreateTrunc(V: IRB.CreateLShr(LHS: R.PtrLong, RHS: PointerTagShift), DestTy: Int8Ty);
846 R.AddrLong = untagPointer(IRB, PtrLong: R.PtrLong);
847 Value *Shadow = memToShadow(Mem: R.AddrLong, IRB);
848 R.MemTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: Shadow);
849 Value *TagMismatch = IRB.CreateICmpNE(LHS: R.PtrTag, RHS: R.MemTag);
850
851 if (MatchAllTag.has_value()) {
852 Value *TagNotIgnored = IRB.CreateICmpNE(
853 LHS: R.PtrTag, RHS: ConstantInt::get(Ty: R.PtrTag->getType(), V: *MatchAllTag));
854 TagMismatch = IRB.CreateAnd(LHS: TagMismatch, RHS: TagNotIgnored);
855 }
856
857 R.TagMismatchTerm = SplitBlockAndInsertIfThen(
858 Cond: TagMismatch, SplitBefore: InsertBefore, Unreachable: false,
859 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
860
861 return R;
862}
863
864void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
865 unsigned AccessSizeIndex,
866 Instruction *InsertBefore,
867 DomTreeUpdater &DTU,
868 LoopInfo *LI) {
869 assert(!UsePageAliases);
870 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
871
872 if (InlineFastPath)
873 InsertBefore =
874 insertShadowTagCheck(Ptr, InsertBefore, DTU, LI).TagMismatchTerm;
875
876 IRBuilder<> IRB(InsertBefore);
877 bool UseFixedShadowIntrinsic = false;
878 // The memaccess fixed shadow intrinsic is only supported on AArch64,
879 // which allows a 16-bit immediate to be left-shifted by 32.
880 // Since kShadowBaseAlignment == 32, and Linux by default will not
881 // mmap above 48-bits, practically any valid shadow offset is
882 // representable.
883 // In particular, an offset of 4TB (1024 << 32) is representable, and
884 // ought to be good enough for anybody.
885 if (TargetTriple.isAArch64() && Mapping.isFixed()) {
886 uint16_t OffsetShifted = Mapping.offset() >> 32;
887 UseFixedShadowIntrinsic =
888 static_cast<uint64_t>(OffsetShifted) << 32 == Mapping.offset();
889 }
890
891 if (UseFixedShadowIntrinsic) {
892 IRB.CreateIntrinsic(
893 ID: UseShortGranules
894 ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
895 : Intrinsic::hwasan_check_memaccess_fixedshadow,
896 Args: {Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo),
897 ConstantInt::get(Ty: Int64Ty, V: Mapping.offset())});
898 } else {
899 IRB.CreateIntrinsic(
900 ID: UseShortGranules ? Intrinsic::hwasan_check_memaccess_shortgranules
901 : Intrinsic::hwasan_check_memaccess,
902 Args: {ShadowBase, Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo)});
903 }
904}
905
906void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
907 unsigned AccessSizeIndex,
908 Instruction *InsertBefore,
909 DomTreeUpdater &DTU,
910 LoopInfo *LI) {
911 assert(!UsePageAliases);
912 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
913
914 ShadowTagCheckInfo TCI = insertShadowTagCheck(Ptr, InsertBefore, DTU, LI);
915
916 IRBuilder<> IRB(TCI.TagMismatchTerm);
917 Value *OutOfShortGranuleTagRange =
918 IRB.CreateICmpUGT(LHS: TCI.MemTag, RHS: ConstantInt::get(Ty: Int8Ty, V: 15));
919 Instruction *CheckFailTerm = SplitBlockAndInsertIfThen(
920 Cond: OutOfShortGranuleTagRange, SplitBefore: TCI.TagMismatchTerm, Unreachable: !Recover,
921 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
922
923 IRB.SetInsertPoint(TCI.TagMismatchTerm);
924 Value *PtrLowBits = IRB.CreateTrunc(V: IRB.CreateAnd(LHS: TCI.PtrLong, RHS: 15), DestTy: Int8Ty);
925 PtrLowBits = IRB.CreateAdd(
926 LHS: PtrLowBits, RHS: ConstantInt::get(Ty: Int8Ty, V: (1 << AccessSizeIndex) - 1));
927 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(LHS: PtrLowBits, RHS: TCI.MemTag);
928 SplitBlockAndInsertIfThen(Cond: PtrLowBitsOOB, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
929 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
930 LI, ThenBlock: CheckFailTerm->getParent());
931
932 IRB.SetInsertPoint(TCI.TagMismatchTerm);
933 Value *InlineTagAddr = IRB.CreateOr(LHS: TCI.AddrLong, RHS: 15);
934 InlineTagAddr = IRB.CreateIntToPtr(V: InlineTagAddr, DestTy: PtrTy);
935 Value *InlineTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: InlineTagAddr);
936 Value *InlineTagMismatch = IRB.CreateICmpNE(LHS: TCI.PtrTag, RHS: InlineTag);
937 SplitBlockAndInsertIfThen(Cond: InlineTagMismatch, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
938 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
939 LI, ThenBlock: CheckFailTerm->getParent());
940
941 IRB.SetInsertPoint(CheckFailTerm);
942 InlineAsm *Asm;
943 switch (TargetTriple.getArch()) {
944 case Triple::x86_64:
945 // The signal handler will find the data address in rdi.
946 Asm = InlineAsm::get(
947 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
948 AsmString: "int3\nnopl " +
949 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
950 "(%rax)",
951 Constraints: "{rdi}",
952 /*hasSideEffects=*/true);
953 break;
954 case Triple::aarch64:
955 case Triple::aarch64_be:
956 // The signal handler will find the data address in x0.
957 Asm = InlineAsm::get(
958 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
959 AsmString: "brk #" + itostr(X: 0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
960 Constraints: "{x0}",
961 /*hasSideEffects=*/true);
962 break;
963 case Triple::riscv64:
964 // The signal handler will find the data address in x10.
965 Asm = InlineAsm::get(
966 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
967 AsmString: "ebreak\naddiw x0, x11, " +
968 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
969 Constraints: "{x10}",
970 /*hasSideEffects=*/true);
971 break;
972 default:
973 report_fatal_error(reason: "unsupported architecture");
974 }
975 IRB.CreateCall(Callee: Asm, Args: TCI.PtrLong);
976 if (Recover)
977 cast<UncondBrInst>(Val: CheckFailTerm)
978 ->setSuccessor(TCI.TagMismatchTerm->getParent());
979}
980
981bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE,
982 MemIntrinsic *MI) {
983 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Val: MI)) {
984 return (!Opts.hwasan_instrument_writes ||
985 ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getDest())) &&
986 (!Opts.hwasan_instrument_reads ||
987 ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getSource()));
988 }
989 if (isa<MemSetInst>(Val: MI))
990 return !Opts.hwasan_instrument_writes ||
991 ignoreAccess(ORE, Inst: MI, Ptr: MI->getDest());
992 return false;
993}
994
995void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
996 IRBuilder<> IRB(MI);
997 if (isa<MemTransferInst>(Val: MI)) {
998 SmallVector<Value *, 4> Args{
999 MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
1000 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1001
1002 if (UseMatchAllCallback)
1003 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1004 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? HwasanMemmove : HwasanMemcpy, Args);
1005 } else if (isa<MemSetInst>(Val: MI)) {
1006 SmallVector<Value *, 4> Args{
1007 MI->getOperand(i_nocapture: 0),
1008 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1009 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1010 if (UseMatchAllCallback)
1011 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1012 IRB.CreateCall(Callee: HwasanMemset, Args);
1013 }
1014 MI->eraseFromParent();
1015}
1016
1017bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O,
1018 DomTreeUpdater &DTU, LoopInfo *LI,
1019 const DataLayout &DL) {
1020 Value *Addr = O.getPtr();
1021
1022 LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
1023
1024 // If the pointer is statically known to be zero, the tag check will pass
1025 // since:
1026 // 1) it has a zero tag
1027 // 2) the shadow memory corresponding to address 0 is initialized to zero and
1028 // never updated.
1029 // We can therefore elide the tag check.
1030 llvm::KnownBits Known(DL.getPointerTypeSizeInBits(Addr->getType()));
1031 llvm::computeKnownBits(V: Addr, Known, DL);
1032 if (Known.isZero())
1033 return false;
1034
1035 if (O.MaybeMask)
1036 return false; // FIXME
1037
1038 IRBuilder<> IRB(O.getInsn());
1039 if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(Value: O.TypeStoreSize) &&
1040 (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1041 (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
1042 *O.Alignment >= O.TypeStoreSize / 8)) {
1043 size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize: O.TypeStoreSize);
1044 if (InstrumentWithCalls) {
1045 SmallVector<Value *, 2> Args{IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy)};
1046 if (UseMatchAllCallback)
1047 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1048 IRB.CreateCall(Callee: HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1049 Args);
1050 } else if (OutlinedChecks) {
1051 instrumentMemAccessOutline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1052 DTU, LI);
1053 } else {
1054 instrumentMemAccessInline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1055 DTU, LI);
1056 }
1057 } else {
1058 SmallVector<Value *, 3> Args{
1059 IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy),
1060 IRB.CreateUDiv(LHS: IRB.CreateTypeSize(Ty: IntptrTy, Size: O.TypeStoreSize),
1061 RHS: ConstantInt::get(Ty: IntptrTy, V: 8))};
1062 if (UseMatchAllCallback)
1063 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1064 IRB.CreateCall(Callee: HwasanMemoryAccessCallbackSized[O.IsWrite], Args);
1065 }
1066 untagPointerOperand(I: O.getInsn(), Addr);
1067
1068 return true;
1069}
1070
1071void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1072 size_t Size) {
1073 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1074 if (!UseShortGranules)
1075 Size = AlignedSize;
1076
1077 Tag = IRB.CreateTrunc(V: Tag, DestTy: Int8Ty);
1078 if (InstrumentWithCalls) {
1079 IRB.CreateCall(Callee: HwasanTagMemoryFunc,
1080 Args: {IRB.CreatePointerCast(V: AI, DestTy: PtrTy), Tag,
1081 ConstantInt::get(Ty: IntptrTy, V: AlignedSize)});
1082 } else {
1083 size_t ShadowSize = Size >> Mapping.scale();
1084 Value *AddrLong = untagPointer(IRB, PtrLong: IRB.CreatePointerCast(V: AI, DestTy: IntptrTy));
1085 Value *ShadowPtr = memToShadow(Mem: AddrLong, IRB);
1086 // If this memset is not inlined, it will be intercepted in the hwasan
1087 // runtime library. That's OK, because the interceptor skips the checks if
1088 // the address is in the shadow region.
1089 // FIXME: the interceptor is not as fast as real memset. Consider lowering
1090 // llvm.memset right here into either a sequence of stores, or a call to
1091 // hwasan_tag_memory.
1092 if (ShadowSize)
1093 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Tag, Size: ShadowSize, Align: Align(1));
1094 if (Size != AlignedSize) {
1095 const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1096 IRB.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: SizeRemainder),
1097 Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: ShadowPtr, Idx0: ShadowSize));
1098 IRB.CreateStore(
1099 Val: Tag, Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: IRB.CreatePointerCast(V: AI, DestTy: PtrTy),
1100 Idx0: AlignedSize - 1));
1101 }
1102 }
1103}
1104
1105unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1106 if (TargetTriple.getArch() == Triple::x86_64)
1107 return AllocaNo & TagMaskByte;
1108
1109 // A list of 8-bit numbers that have at most one run of non-zero bits.
1110 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1111 // masks.
1112 // The list does not include the value 255, which is used for UAR.
1113 //
1114 // Because we are more likely to use earlier elements of this list than later
1115 // ones, it is sorted in increasing order of probability of collision with a
1116 // mask allocated (temporally) nearby. The program that generated this list
1117 // can be found at:
1118 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1119 static const unsigned FastMasks[] = {
1120 0, 128, 64, 192, 32, 96, 224, 112, 240, 48, 16, 120,
1121 248, 56, 24, 8, 124, 252, 60, 28, 12, 4, 126, 254,
1122 62, 30, 14, 6, 2, 127, 63, 31, 15, 7, 3, 1};
1123 return FastMasks[AllocaNo % std::size(FastMasks)];
1124}
1125
1126Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1127 if (TagMaskByte == 0xFF)
1128 return OldTag; // No need to clear the tag byte.
1129 return IRB.CreateAnd(LHS: OldTag,
1130 RHS: ConstantInt::get(Ty: OldTag->getType(), V: TagMaskByte));
1131}
1132
1133Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1134 return IRB.CreateZExt(V: IRB.CreateCall(Callee: HwasanGenerateTagFunc), DestTy: IntptrTy);
1135}
1136
1137Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1138 if (Opts.hwasan_generate_tags_with_calls)
1139 return nullptr;
1140 if (StackBaseTag)
1141 return StackBaseTag;
1142 // Extract some entropy from the stack pointer for the tags.
1143 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1144 // between functions).
1145 Value *FramePointerLong = getCachedFP(IRB);
1146 Value *StackTag =
1147 applyTagMask(IRB, OldTag: IRB.CreateXor(LHS: FramePointerLong,
1148 RHS: IRB.CreateLShr(LHS: FramePointerLong, RHS: 20)));
1149 StackTag->setName("hwasan.stack.base.tag");
1150 return StackTag;
1151}
1152
1153Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1154 unsigned AllocaNo) {
1155 if (Opts.hwasan_generate_tags_with_calls)
1156 return getNextTagWithCall(IRB);
1157 return IRB.CreateXor(
1158 LHS: StackTag, RHS: ConstantInt::get(Ty: StackTag->getType(), V: retagMask(AllocaNo)));
1159}
1160
1161Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
1162 Value *FramePointerLong = getCachedFP(IRB);
1163 Value *UARTag =
1164 applyTagMask(IRB, OldTag: IRB.CreateLShr(LHS: FramePointerLong, RHS: PointerTagShift));
1165
1166 UARTag->setName("hwasan.uar.tag");
1167 return UARTag;
1168}
1169
1170// Add a tag to an address.
1171Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1172 Value *PtrLong, Value *Tag) {
1173 assert(!UsePageAliases);
1174 Value *TaggedPtrLong;
1175 if (CompileKernel) {
1176 // Kernel addresses have 0xFF in the most significant byte.
1177 Value *ShiftedTag =
1178 IRB.CreateOr(LHS: IRB.CreateShl(LHS: Tag, RHS: PointerTagShift),
1179 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << PointerTagShift) - 1));
1180 TaggedPtrLong = IRB.CreateAnd(LHS: PtrLong, RHS: ShiftedTag);
1181 } else {
1182 // Userspace can simply do OR (tag << PointerTagShift);
1183 Value *ShiftedTag = IRB.CreateShl(LHS: Tag, RHS: PointerTagShift);
1184 TaggedPtrLong = IRB.CreateOr(LHS: PtrLong, RHS: ShiftedTag);
1185 }
1186 return IRB.CreateIntToPtr(V: TaggedPtrLong, DestTy: Ty);
1187}
1188
1189// Remove tag from an address.
1190Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1191 assert(!UsePageAliases);
1192 Value *UntaggedPtrLong;
1193 if (CompileKernel) {
1194 // Kernel addresses have 0xFF in the most significant byte.
1195 UntaggedPtrLong =
1196 IRB.CreateOr(LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1197 V: TagMaskByte << PointerTagShift));
1198 } else {
1199 // Userspace addresses have 0x00.
1200 UntaggedPtrLong = IRB.CreateAnd(
1201 LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1202 V: ~(TagMaskByte << PointerTagShift)));
1203 }
1204 return UntaggedPtrLong;
1205}
1206
1207Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) {
1208 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1209 // in Bionic's libc/platform/bionic/tls_defines.h.
1210 constexpr int SanitizerSlot = 6;
1211 if (TargetTriple.isAArch64() && TargetTriple.isAndroid())
1212 return memtag::getAndroidSlotPtr(IRB, Slot: SanitizerSlot);
1213 return ThreadPtrGlobal;
1214}
1215
1216Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) {
1217 if (!CachedFP)
1218 CachedFP = memtag::getFP(IRB);
1219 return CachedFP;
1220}
1221
1222Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1223 // Prepare ring buffer data.
1224 Value *PC = memtag::getPC(TargetTriple, IRB);
1225 Value *FP = getCachedFP(IRB);
1226
1227 // Mix FP and PC.
1228 // Assumptions:
1229 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
1230 // FP is 0xfffffffffffFFFF0 (4 lower bits are zero)
1231 // We only really need ~20 lower non-zero bits (FFFF), so we mix like this:
1232 // 0xFFFFPPPPPPPPPPPP
1233 //
1234 // FP works because in AArch64FrameLowering::getFrameIndexReference, we
1235 // prefer FP-relative offsets for functions compiled with HWASan.
1236 FP = IRB.CreateShl(LHS: FP, RHS: 44);
1237 return IRB.CreateOr(LHS: PC, RHS: FP);
1238}
1239
1240void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1241 if (!Mapping.isInTls())
1242 ShadowBase = getShadowNonTls(IRB);
1243 else if (!WithFrameRecord && TargetTriple.isAndroid())
1244 ShadowBase = getDynamicShadowIfunc(IRB);
1245
1246 if (!WithFrameRecord && ShadowBase)
1247 return;
1248
1249 Value *SlotPtr = nullptr;
1250 Value *ThreadLong = nullptr;
1251 Value *ThreadLongMaybeUntagged = nullptr;
1252
1253 auto getThreadLongMaybeUntagged = [&]() {
1254 if (!SlotPtr)
1255 SlotPtr = getHwasanThreadSlotPtr(IRB);
1256 if (!ThreadLong)
1257 ThreadLong = IRB.CreateLoad(Ty: IntptrTy, Ptr: SlotPtr);
1258 // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1259 // TBI.
1260 return TargetTriple.isAArch64() ? ThreadLong
1261 : untagPointer(IRB, PtrLong: ThreadLong);
1262 };
1263
1264 if (WithFrameRecord) {
1265 switch (Opts.hwasan_record_stack_history) {
1266 case RecordStackHistoryMode::libcall: {
1267 // Emit a runtime call into hwasan rather than emitting instructions for
1268 // recording stack history.
1269 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1270 IRB.CreateCall(Callee: HwasanRecordFrameRecordFunc, Args: {FrameRecordInfo});
1271 break;
1272 }
1273 case RecordStackHistoryMode::instr: {
1274 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1275
1276 StackBaseTag = IRB.CreateAShr(LHS: ThreadLong, RHS: 3);
1277
1278 // Store data to ring buffer.
1279 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1280 Value *RecordPtr =
1281 IRB.CreateIntToPtr(V: ThreadLongMaybeUntagged, DestTy: IRB.getPtrTy(AddrSpace: 0));
1282 IRB.CreateStore(Val: FrameRecordInfo, Ptr: RecordPtr);
1283
1284 IRB.CreateStore(Val: memtag::incrementThreadLong(IRB, ThreadLong, Inc: 8), Ptr: SlotPtr);
1285 break;
1286 }
1287 case RecordStackHistoryMode::none: {
1288 llvm_unreachable(
1289 "A stack history recording mode should've been selected.");
1290 }
1291 }
1292 }
1293
1294 if (!ShadowBase) {
1295 if (!ThreadLongMaybeUntagged)
1296 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1297
1298 // Get shadow base address by aligning RecordPtr up.
1299 // Note: this is not correct if the pointer is already aligned.
1300 // Runtime library will make sure this never happens.
1301 ShadowBase = IRB.CreateAdd(
1302 LHS: IRB.CreateOr(
1303 LHS: ThreadLongMaybeUntagged,
1304 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << kShadowBaseAlignment) - 1)),
1305 RHS: ConstantInt::get(Ty: IntptrTy, V: 1), Name: "hwasan.shadow");
1306 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: PtrTy);
1307 }
1308}
1309
1310void HWAddressSanitizer::instrumentLandingPads(
1311 SmallVectorImpl<Instruction *> &LandingPadVec) {
1312 for (auto *LP : LandingPadVec) {
1313 IRBuilder<> IRB(LP->getNextNode());
1314 IRB.CreateCall(
1315 Callee: HwasanHandleVfork,
1316 Args: {memtag::readRegister(
1317 IRB, Name: (TargetTriple.getArch() == Triple::x86_64) ? "rsp" : "sp")});
1318 }
1319}
1320
1321void HWAddressSanitizer::instrumentStack(OptimizationRemarkEmitter &ORE,
1322 memtag::StackInfo &SInfo,
1323 Value *StackTag, Value *UARTag,
1324 const DominatorTree &DT,
1325 const PostDominatorTree &PDT,
1326 const LoopInfo &LI) {
1327 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1328 // alloca addresses using that. Unfortunately, offsets are not known yet
1329 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1330 // temp, shift-OR it into each alloca address and xor with the retag mask.
1331 // This generates one extra instruction per alloca use.
1332 unsigned int I = 0;
1333
1334 for (auto &KV : SInfo.AllocasToInstrument) {
1335 auto N = I++;
1336 auto *AI = KV.first;
1337 memtag::AllocaInfo &Info = KV.second;
1338 IRBuilder<> IRB(AI->getNextNode());
1339 IRB.disableGEPCanonicalization();
1340
1341 // Replace uses of the alloca with tagged address.
1342 Value *Tag = getAllocaTag(IRB, StackTag, AllocaNo: N);
1343 Value *AILong = IRB.CreatePointerCast(V: AI, DestTy: IntptrTy);
1344 Value *AINoTagLong = untagPointer(IRB, PtrLong: AILong);
1345 Value *Replacement = tagPointer(IRB, Ty: AI->getType(), PtrLong: AINoTagLong, Tag);
1346 std::string Name =
1347 AI->hasName() ? AI->getName().str() : "alloca." + itostr(X: N);
1348 Replacement->setName(Name + ".hwasan");
1349
1350 size_t Size = memtag::getAllocaSizeInBytes(AI: *AI);
1351 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1352
1353 AI->replaceUsesWithIf(New: Replacement, ShouldReplace: [AILong](const Use &U) {
1354 auto *User = U.getUser();
1355 return User != AILong && !isa<LifetimeIntrinsic>(Val: User);
1356 });
1357
1358 memtag::annotateDebugRecords(Info, Tag: retagMask(AllocaNo: N));
1359
1360 auto TagStarts = [&]() {
1361 for (IntrinsicInst *Start : Info.LifetimeStart) {
1362 IRB.SetInsertPoint(Start->getNextNode());
1363 tagAlloca(IRB, AI, Tag, Size);
1364 }
1365 };
1366 auto TagEnd = [&](Instruction *Node) {
1367 IRB.SetInsertPoint(Node);
1368 // When untagging, use the `AlignedSize` because we need to set the tags
1369 // for the entire alloca to original. If we used `Size` here, we would
1370 // keep the last granule tagged, and store zero in the last byte of the
1371 // last granule, due to how short granules are implemented.
1372 tagAlloca(IRB, AI, Tag: UARTag, Size: AlignedSize);
1373 };
1374 auto EraseLifetimes = [&]() {
1375 for (auto &II : Info.LifetimeStart)
1376 II->eraseFromParent();
1377 for (auto &II : Info.LifetimeEnd)
1378 II->eraseFromParent();
1379 };
1380 // Calls to functions that may return twice (e.g. setjmp) confuse the
1381 // postdominator analysis, and will leave us to keep memory tagged after
1382 // function return. Work around this by always untagging at every return
1383 // statement if return_twice functions are called.
1384 if (DetectUseAfterScope && !SInfo.CallsReturnTwice &&
1385 memtag::isSupportedLifetime(AInfo: Info, DT: &DT, LI: &LI)) {
1386 TagStarts();
1387 memtag::forAllReachableExits(DT, PDT, LI, AInfo: Info, RetVec: SInfo.RetVec, Callback: TagEnd);
1388 ORE.emit(RemarkBuilder: [&]() {
1389 return OptimizationRemark(DEBUG_TYPE, "supportedLifetime", AI);
1390 });
1391 } else if (DetectUseAfterScope && Opts.hwasan_strict_use_after_scope) {
1392 // SInfo.CallsReturnTwice || !isStandardLifetime
1393 ORE.emit(RemarkBuilder: [&]() {
1394 return OptimizationRemarkMissed(DEBUG_TYPE, "supportedLifetime", AI);
1395 });
1396
1397 tagAlloca(IRB, AI, Tag, Size);
1398 TagStarts();
1399 for_each(Range&: Info.LifetimeEnd, F: TagEnd);
1400 for_each(Range&: SInfo.RetVec, F: TagEnd);
1401 EraseLifetimes();
1402 } else {
1403 tagAlloca(IRB, AI, Tag, Size);
1404 for_each(Range&: SInfo.RetVec, F: TagEnd);
1405 EraseLifetimes();
1406 }
1407 memtag::alignAndPadAlloca(Info, Align: Mapping.getObjectAlignment());
1408 }
1409}
1410
1411static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE,
1412 bool Skip) {
1413 if (Skip) {
1414 ORE.emit(RemarkBuilder: [&]() {
1415 return OptimizationRemark(DEBUG_TYPE, "Skip", &F)
1416 << "Skipped: F=" << ore::NV("Function", &F);
1417 });
1418 } else {
1419 ORE.emit(RemarkBuilder: [&]() {
1420 return OptimizationRemarkMissed(DEBUG_TYPE, "Sanitize", &F)
1421 << "Sanitized: F=" << ore::NV("Function", &F);
1422 });
1423 }
1424}
1425
1426bool HWAddressSanitizer::selectiveInstrumentationShouldSkip(
1427 Function &F, FunctionAnalysisManager &FAM) const {
1428 auto SkipHot = [&]() {
1429 if (!Opts.hwasan_percentile_cutoff_hot)
1430 return false;
1431 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1432 ProfileSummaryInfo *PSI =
1433 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
1434 if (!PSI || !PSI->hasProfileSummary()) {
1435 ++NumNoProfileSummaryFuncs;
1436 return false;
1437 }
1438 return PSI->isFunctionHotInCallGraphNthPercentile(
1439 PercentileCutoff: *Opts.hwasan_percentile_cutoff_hot, F: &F,
1440 BFI&: FAM.getResult<BlockFrequencyAnalysis>(IR&: F));
1441 };
1442
1443 auto SkipRandom = [&]() {
1444 if (!Opts.hwasan_random_rate)
1445 return false;
1446 std::bernoulli_distribution D(*Opts.hwasan_random_rate);
1447 return !D(*Rng);
1448 };
1449
1450 bool Skip = SkipRandom() || SkipHot();
1451 emitRemark(F, ORE&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F), Skip);
1452 return Skip;
1453}
1454
1455void HWAddressSanitizer::sanitizeFunction(Function &F,
1456 FunctionAnalysisManager &FAM) {
1457 if (&F == HwasanCtorFunction)
1458 return;
1459
1460 // Do not apply any instrumentation for naked functions.
1461 if (F.hasFnAttribute(Kind: Attribute::Naked))
1462 return;
1463
1464 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
1465 return;
1466
1467 if (!F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1468 return;
1469
1470 if (F.empty())
1471 return;
1472
1473 if (F.isPresplitCoroutine())
1474 return;
1475
1476 NumTotalFuncs++;
1477
1478 OptimizationRemarkEmitter &ORE =
1479 FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
1480
1481 if (selectiveInstrumentationShouldSkip(F, FAM))
1482 return;
1483
1484 NumInstrumentedFuncs++;
1485
1486 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1487
1488 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1489 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1490 SmallVector<Instruction *, 8> LandingPadVec;
1491 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1492
1493 memtag::StackInfoBuilder SIB(SSI, DEBUG_TYPE);
1494 for (auto &Inst : instructions(F)) {
1495 if (InstrumentStack) {
1496 SIB.visit(ORE, Inst);
1497 }
1498
1499 if (InstrumentLandingPads && isa<LandingPadInst>(Val: Inst))
1500 LandingPadVec.push_back(Elt: &Inst);
1501
1502 getInterestingMemoryOperands(ORE, I: &Inst, TLI, Interesting&: OperandsToInstrument);
1503
1504 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst))
1505 if (!ignoreMemIntrinsic(ORE, MI))
1506 IntrinToInstrument.push_back(Elt: MI);
1507 }
1508
1509 memtag::StackInfo &SInfo = SIB.get();
1510
1511 initializeCallbacks(M&: *F.getParent());
1512
1513 if (!LandingPadVec.empty())
1514 instrumentLandingPads(LandingPadVec);
1515
1516 if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1517 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1518 // __hwasan_personality_thunk is a no-op for functions without an
1519 // instrumented stack, so we can drop it.
1520 F.setPersonalityFn(nullptr);
1521 }
1522
1523 if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1524 IntrinToInstrument.empty())
1525 return;
1526
1527 assert(!ShadowBase);
1528
1529 BasicBlock::iterator InsertPt = F.getEntryBlock().begin();
1530 IRBuilder<> EntryIRB(InsertPt);
1531 EntryIRB.disableGEPCanonicalization();
1532 emitPrologue(IRB&: EntryIRB,
1533 /*WithFrameRecord*/ Opts.hwasan_record_stack_history !=
1534 RecordStackHistoryMode::none &&
1535 Mapping.withFrameRecord() &&
1536 !SInfo.AllocasToInstrument.empty());
1537
1538 if (!SInfo.AllocasToInstrument.empty()) {
1539 const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
1540 const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
1541 const LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: F);
1542 Value *StackTag = getStackBaseTag(IRB&: EntryIRB);
1543 Value *UARTag = getUARTag(IRB&: EntryIRB);
1544 // instrumentStack may replace allocas, invalidating EntryIRB's insert
1545 // point. Save the insert block before calling it.
1546 BasicBlock *EntryBB = EntryIRB.GetInsertBlock();
1547 instrumentStack(ORE, SInfo, StackTag, UARTag, DT, PDT, LI);
1548
1549 // If we split the entry block, move any allocas that were originally in the
1550 // entry block back into the entry block so that they aren't treated as
1551 // dynamic allocas.
1552 if (EntryBB != &F.getEntryBlock()) {
1553 InsertPt = F.getEntryBlock().begin();
1554 for (Instruction &I : llvm::make_early_inc_range(Range&: *EntryBB)) {
1555 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
1556 if (isa<ConstantInt>(Val: AI->getArraySize()))
1557 I.moveBefore(BB&: F.getEntryBlock(), I: InsertPt);
1558 }
1559 }
1560 }
1561
1562 DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
1563 PostDominatorTree *PDT = FAM.getCachedResult<PostDominatorTreeAnalysis>(IR&: F);
1564 LoopInfo *LI = FAM.getCachedResult<LoopAnalysis>(IR&: F);
1565 DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
1566 const DataLayout &DL = F.getDataLayout();
1567 for (auto &Operand : OperandsToInstrument)
1568 instrumentMemAccess(O&: Operand, DTU, LI, DL);
1569 DTU.flush();
1570
1571 if (Opts.hwasan_instrument_mem_intrinsics && !IntrinToInstrument.empty()) {
1572 for (auto *Inst : IntrinToInstrument)
1573 instrumentMemIntrinsic(MI: Inst);
1574 }
1575
1576 ShadowBase = nullptr;
1577 StackBaseTag = nullptr;
1578 CachedFP = nullptr;
1579}
1580
1581void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1582 assert(!UsePageAliases);
1583 Constant *Initializer = GV->getInitializer();
1584 uint64_t SizeInBytes =
1585 M.getDataLayout().getTypeAllocSize(Ty: Initializer->getType());
1586 uint64_t NewSize = alignTo(Size: SizeInBytes, A: Mapping.getObjectAlignment());
1587 if (SizeInBytes != NewSize) {
1588 // Pad the initializer out to the next multiple of 16 bytes and add the
1589 // required short granule tag.
1590 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1591 Init.back() = Tag;
1592 Constant *Padding = ConstantDataArray::get(Context&: *C, Elts&: Init);
1593 Initializer = ConstantStruct::getAnon(V: {Initializer, Padding});
1594 }
1595
1596 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1597 GlobalValue::ExternalLinkage, Initializer,
1598 GV->getName() + ".hwasan");
1599 NewGV->copyAttributesFrom(Src: GV);
1600 NewGV->setLinkage(GlobalValue::PrivateLinkage);
1601 NewGV->copyMetadata(Src: GV, Offset: 0);
1602 NewGV->setAlignment(
1603 std::max(a: GV->getAlign().valueOrOne(), b: Mapping.getObjectAlignment()));
1604
1605 // It is invalid to ICF two globals that have different tags. In the case
1606 // where the size of the global is a multiple of the tag granularity the
1607 // contents of the globals may be the same but the tags (i.e. symbol values)
1608 // may be different, and the symbols are not considered during ICF. In the
1609 // case where the size is not a multiple of the granularity, the short granule
1610 // tags would discriminate two globals with different tags, but there would
1611 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1612 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1613 // granule tag in the last byte.
1614 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1615
1616 // Descriptor format (assuming little-endian):
1617 // bytes 0-3: relative address of global
1618 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1619 // it isn't, we create multiple descriptors)
1620 // byte 7: tag
1621 auto *DescriptorTy = StructType::get(elt1: Int32Ty, elts: Int32Ty);
1622 const uint64_t MaxDescriptorSize = 0xfffff0;
1623 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1624 DescriptorPos += MaxDescriptorSize) {
1625 auto *Descriptor =
1626 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1627 nullptr, GV->getName() + ".hwasan.descriptor");
1628 auto *GVRelPtr = ConstantExpr::getTrunc(
1629 C: ConstantExpr::getAdd(
1630 C1: ConstantExpr::getSub(
1631 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1632 C2: ConstantExpr::getPtrToInt(C: Descriptor, Ty: Int64Ty)),
1633 C2: ConstantInt::get(Ty: Int64Ty, V: DescriptorPos)),
1634 Ty: Int32Ty);
1635 uint32_t Size = std::min(a: SizeInBytes - DescriptorPos, b: MaxDescriptorSize);
1636 auto *SizeAndTag = ConstantInt::get(Ty: Int32Ty, V: Size | (uint32_t(Tag) << 24));
1637 Descriptor->setComdat(NewGV->getComdat());
1638 Descriptor->setInitializer(ConstantStruct::getAnon(V: {GVRelPtr, SizeAndTag}));
1639 Descriptor->setSection("hwasan_globals");
1640 Descriptor->setMetadata(KindID: LLVMContext::MD_associated,
1641 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: NewGV)));
1642 appendToCompilerUsed(M, Values: Descriptor);
1643 }
1644
1645 Constant *Aliasee = ConstantExpr::getIntToPtr(
1646 C: ConstantExpr::getAdd(
1647 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1648 C2: ConstantInt::get(Ty: Int64Ty, V: uint64_t(Tag) << PointerTagShift)),
1649 Ty: GV->getType());
1650 auto *Alias = GlobalAlias::create(Ty: GV->getValueType(), AddressSpace: GV->getAddressSpace(),
1651 Linkage: GV->getLinkage(), Name: "", Aliasee, Parent: &M);
1652 Alias->setVisibility(GV->getVisibility());
1653 Alias->takeName(V: GV);
1654 GV->replaceAllUsesWith(V: Alias);
1655 GV->eraseFromParent();
1656}
1657
1658void HWAddressSanitizer::instrumentGlobals() {
1659 std::vector<GlobalVariable *> Globals;
1660 for (GlobalVariable &GV : M.globals()) {
1661 if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1662 continue;
1663
1664 if (GV.isDeclarationForLinker() || GV.getName().starts_with(Prefix: "llvm.") ||
1665 GV.isThreadLocal())
1666 continue;
1667
1668 // Common symbols can't have aliases point to them, so they can't be tagged.
1669 if (GV.hasCommonLinkage())
1670 continue;
1671
1672 if (Opts.hwasan_all_globals) {
1673 // Avoid instrumenting intrinsic global variables.
1674 if (GV.getSection() == "llvm.metadata")
1675 continue;
1676 } else {
1677 // Globals with custom sections may be used in __start_/__stop_
1678 // enumeration, which would be broken both by adding tags and potentially
1679 // by the extra padding/alignment that we insert.
1680 if (GV.hasSection())
1681 continue;
1682 }
1683
1684 Globals.push_back(x: &GV);
1685 }
1686
1687 MD5 Hasher;
1688 Hasher.update(Str: M.getSourceFileName());
1689 MD5::MD5Result Hash;
1690 Hasher.final(Result&: Hash);
1691 uint8_t Tag = Hash[0];
1692
1693 assert(TagMaskByte >= 16);
1694
1695 for (GlobalVariable *GV : Globals) {
1696 // Don't allow globals to be tagged with something that looks like a
1697 // short-granule tag, otherwise we lose inter-granule overflow detection, as
1698 // the fast path shadow-vs-address check succeeds.
1699 if (Tag < 16 || Tag > TagMaskByte)
1700 Tag = 16;
1701 instrumentGlobal(GV, Tag: Tag++);
1702 }
1703}
1704
1705void HWAddressSanitizer::instrumentPersonalityFunctions() {
1706 // We need to untag stack frames as we unwind past them. That is the job of
1707 // the personality function wrapper, which either wraps an existing
1708 // personality function or acts as a personality function on its own. Each
1709 // function that has a personality function or that can be unwound past has
1710 // its personality function changed to a thunk that calls the personality
1711 // function wrapper in the runtime.
1712 MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1713 for (Function &F : M) {
1714 if (F.isDeclaration() || !F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress) ||
1715 F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
1716 continue;
1717
1718 if (F.hasPersonalityFn()) {
1719 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(x: &F);
1720 } else if (!F.hasFnAttribute(Kind: Attribute::NoUnwind)) {
1721 PersonalityFns[nullptr].push_back(x: &F);
1722 }
1723 }
1724
1725 if (PersonalityFns.empty())
1726 return;
1727
1728 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1729 Name: "__hwasan_personality_wrapper", RetTy: Int32Ty, Args: Int32Ty, Args: Int32Ty, Args: Int64Ty, Args: PtrTy,
1730 Args: PtrTy, Args: PtrTy, Args: PtrTy, Args: PtrTy);
1731 FunctionCallee UnwindGetGR = M.getOrInsertFunction(Name: "_Unwind_GetGR", RetTy: VoidTy);
1732 FunctionCallee UnwindGetCFA = M.getOrInsertFunction(Name: "_Unwind_GetCFA", RetTy: VoidTy);
1733
1734 for (auto &P : PersonalityFns) {
1735 std::string ThunkName = kHwasanPersonalityThunkName;
1736 if (P.first)
1737 ThunkName += ("." + P.first->getName()).str();
1738 FunctionType *ThunkFnTy = FunctionType::get(
1739 Result: Int32Ty, Params: {Int32Ty, Int32Ty, Int64Ty, PtrTy, PtrTy}, isVarArg: false);
1740 bool IsLocal = P.first && (!isa<GlobalValue>(Val: P.first) ||
1741 cast<GlobalValue>(Val: P.first)->hasLocalLinkage());
1742 auto *ThunkFn = Function::Create(Ty: ThunkFnTy,
1743 Linkage: IsLocal ? GlobalValue::InternalLinkage
1744 : GlobalValue::LinkOnceODRLinkage,
1745 N: ThunkName, M: &M);
1746 // TODO: think about other attributes as well.
1747 if (any_of(Range&: P.second, P: [](const Function *F) {
1748 return F->hasFnAttribute(Kind: "branch-target-enforcement");
1749 })) {
1750 ThunkFn->addFnAttr(Kind: "branch-target-enforcement");
1751 }
1752 if (!IsLocal) {
1753 ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1754 ThunkFn->setComdat(M.getOrInsertComdat(Name: ThunkName));
1755 }
1756
1757 auto *BB = BasicBlock::Create(Context&: *C, Name: "entry", Parent: ThunkFn);
1758 IRBuilder<> IRB(BB);
1759 CallInst *WrapperCall = IRB.CreateCall(
1760 Callee: HwasanPersonalityWrapper,
1761 Args: {ThunkFn->getArg(i: 0), ThunkFn->getArg(i: 1), ThunkFn->getArg(i: 2),
1762 ThunkFn->getArg(i: 3), ThunkFn->getArg(i: 4),
1763 P.first ? P.first : Constant::getNullValue(Ty: PtrTy),
1764 UnwindGetGR.getCallee(), UnwindGetCFA.getCallee()});
1765 WrapperCall->setTailCall();
1766 IRB.CreateRet(V: WrapperCall);
1767
1768 for (Function *F : P.second)
1769 F->setPersonalityFn(ThunkFn);
1770 }
1771}
1772
1773void HWAddressSanitizer::ShadowMapping::init(const InstrumentationOptions &Opts,
1774 Triple &TargetTriple,
1775 bool InstrumentWithCalls,
1776 bool CompileKernel) {
1777 // Start with defaults.
1778 Scale = kDefaultShadowScale;
1779 Kind = OffsetKind::kTls;
1780 WithFrameRecord = true;
1781
1782 // Tune for the target.
1783 if (TargetTriple.isOSFuchsia()) {
1784 // Fuchsia is always PIE, which means that the beginning of the address
1785 // space is always available.
1786 Kind = OffsetKind::kGlobal;
1787 } else if (CompileKernel || InstrumentWithCalls) {
1788 SetFixed(0);
1789 WithFrameRecord = false;
1790 }
1791
1792 WithFrameRecord = valueOr(X: Opts.hwasan_with_frame_record, Default: WithFrameRecord);
1793
1794 // Apply the last of ClMappingOffset and ClMappingOffsetDynamic.
1795 if (ClMappingOffsetDynamic.getNumOccurrences())
1796 Kind = ClMappingOffsetDynamic;
1797 if (ClMappingOffset.getNumOccurrences() > 0 &&
1798 !(ClMappingOffsetDynamic.getNumOccurrences() > 0 &&
1799 ClMappingOffsetDynamic.getPosition() > ClMappingOffset.getPosition())) {
1800 SetFixed(ClMappingOffset);
1801 }
1802}
1803