1//===- HWAddressSanitizer.cpp - memory access error detector --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of HWAddressSanitizer, an address basic correctness
11/// checker based on tagged addressing.
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15#include "InstrumentationOptions.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/BlockFrequencyInfo.h"
23#include "llvm/Analysis/DomTreeUpdater.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/OptimizationRemarkEmitter.h"
26#include "llvm/Analysis/PostDominators.h"
27#include "llvm/Analysis/ProfileSummaryInfo.h"
28#include "llvm/Analysis/StackSafetyAnalysis.h"
29#include "llvm/Analysis/TargetLibraryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/BinaryFormat/ELF.h"
33#include "llvm/IR/Attributes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/DerivedTypes.h"
39#include "llvm/IR/Dominators.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/LLVMContext.h"
49#include "llvm/IR/MDBuilder.h"
50#include "llvm/IR/Module.h"
51#include "llvm/IR/Type.h"
52#include "llvm/IR/Value.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/MD5.h"
58#include "llvm/Support/RandomNumberGenerator.h"
59#include "llvm/Support/raw_ostream.h"
60#include "llvm/TargetParser/Triple.h"
61#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
62#include "llvm/Transforms/Utils/BasicBlockUtils.h"
63#include "llvm/Transforms/Utils/Instrumentation.h"
64#include "llvm/Transforms/Utils/Local.h"
65#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
66#include "llvm/Transforms/Utils/ModuleUtils.h"
67#include "llvm/Transforms/Utils/PromoteMemToReg.h"
68#include <optional>
69#include <random>
70
71using namespace llvm;
72
73#define DEBUG_TYPE "hwasan"
74
75const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
76const char kHwasanNoteName[] = "hwasan.note";
77const char kHwasanInitName[] = "__hwasan_init";
78const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
79
80const char kHwasanShadowMemoryDynamicAddress[] =
81 "__hwasan_shadow_memory_dynamic_address";
82
83// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
84static const size_t kNumberOfAccessSizes = 5;
85
86static const size_t kDefaultShadowScale = 4;
87
88static const unsigned kShadowBaseAlignment = 32;
89
90namespace {
91enum class OffsetKind {
92 kFixed = 0,
93 kGlobal,
94 kIfunc,
95 kTls,
96};
97}
98
99// These flags allow to change the shadow mapping and control how shadow memory
100// is accessed. The shadow mapping looks like:
101// Shadow = (Mem >> scale) + offset
102
103static cl::opt<uint64_t>
104 ClMappingOffset("hwasan-mapping-offset",
105 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
106 cl::Hidden);
107
108static cl::opt<OffsetKind> ClMappingOffsetDynamic(
109 "hwasan-mapping-offset-dynamic",
110 cl::desc("HWASan shadow mapping dynamic offset location"), cl::Hidden,
111 cl::values(clEnumValN(OffsetKind::kGlobal, "global", "Use global"),
112 clEnumValN(OffsetKind::kIfunc, "ifunc", "Use ifunc global"),
113 clEnumValN(OffsetKind::kTls, "tls", "Use TLS")));
114
115STATISTIC(NumTotalFuncs, "Number of total funcs");
116STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs");
117STATISTIC(NumNoProfileSummaryFuncs, "Number of funcs without PS");
118
119namespace {
120
121bool shouldUsePageAliases(const InstrumentationOptions &Opts,
122 const Triple &TargetTriple) {
123 return Opts.hwasan_experimental_use_page_aliases &&
124 TargetTriple.getArch() == Triple::x86_64;
125}
126
127bool shouldInstrumentStack(const InstrumentationOptions &Opts,
128 const Triple &TargetTriple) {
129 return !shouldUsePageAliases(Opts, TargetTriple) &&
130 Opts.hwasan_instrument_stack;
131}
132
133bool shouldInstrumentWithCalls(const InstrumentationOptions &Opts,
134 const Triple &TargetTriple) {
135 return valueOr(X: Opts.hwasan_instrument_with_calls,
136 Default: TargetTriple.getArch() == Triple::x86_64);
137}
138
139bool mightUseStackSafetyAnalysis(const InstrumentationOptions &Opts,
140 bool DisableOptimization) {
141 return valueOr(X: Opts.hwasan_use_stack_safety, Default: !DisableOptimization);
142}
143
144bool shouldUseStackSafetyAnalysis(const InstrumentationOptions &Opts,
145 const Triple &TargetTriple,
146 bool DisableOptimization) {
147 return shouldInstrumentStack(Opts, TargetTriple) &&
148 mightUseStackSafetyAnalysis(Opts, DisableOptimization);
149}
150
151bool shouldDetectUseAfterScope(const InstrumentationOptions &Opts,
152 const Triple &TargetTriple) {
153 return Opts.hwasan_use_after_scope &&
154 shouldInstrumentStack(Opts, TargetTriple);
155}
156
157/// An instrumentation pass implementing detection of addressability bugs
158/// using tagged pointers.
159class HWAddressSanitizer {
160public:
161 HWAddressSanitizer(const InstrumentationOptions &Opts, Module &M,
162 bool CompileKernel, bool Recover,
163 const StackSafetyGlobalInfo *SSI)
164 : Opts(Opts), M(M), SSI(SSI) {
165 this->Recover = valueOr(X: Opts.hwasan_recover, Default: Recover);
166 this->CompileKernel = valueOr(X: Opts.hwasan_kernel, Default: CompileKernel);
167 this->Rng = Opts.hwasan_random_rate ? M.createRNG(DEBUG_TYPE) : nullptr;
168
169 initializeModule();
170 }
171
172 void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
173
174private:
175 struct ShadowTagCheckInfo {
176 Instruction *TagMismatchTerm = nullptr;
177 Value *PtrLong = nullptr;
178 Value *AddrLong = nullptr;
179 Value *PtrTag = nullptr;
180 Value *MemTag = nullptr;
181 };
182
183 bool selectiveInstrumentationShouldSkip(Function &F,
184 FunctionAnalysisManager &FAM) const;
185 void initializeModule();
186 void createHwasanCtorComdat();
187 void createHwasanNote();
188
189 void initializeCallbacks(Module &M);
190
191 Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
192
193 Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
194 Value *getShadowNonTls(IRBuilder<> &IRB);
195
196 void untagPointerOperand(Instruction *I, Value *Addr);
197 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
198
199 int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
200 ShadowTagCheckInfo insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
201 DomTreeUpdater &DTU, LoopInfo *LI);
202 void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
203 unsigned AccessSizeIndex,
204 Instruction *InsertBefore,
205 DomTreeUpdater &DTU, LoopInfo *LI);
206 void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
207 unsigned AccessSizeIndex,
208 Instruction *InsertBefore, DomTreeUpdater &DTU,
209 LoopInfo *LI);
210 bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI);
211 void instrumentMemIntrinsic(MemIntrinsic *MI);
212 bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
213 LoopInfo *LI, const DataLayout &DL);
214 bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr);
215 bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst,
216 Value *Ptr);
217
218 void getInterestingMemoryOperands(
219 OptimizationRemarkEmitter &ORE, Instruction *I,
220 const TargetLibraryInfo &TLI,
221 SmallVectorImpl<InterestingMemoryOperand> &Interesting);
222
223 void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
224 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
225 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
226 void instrumentStack(OptimizationRemarkEmitter &ORE, memtag::StackInfo &Info,
227 Value *StackTag, Value *UARTag, const DominatorTree &DT,
228 const PostDominatorTree &PDT, const LoopInfo &LI);
229 void instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
230 Value *getNextTagWithCall(IRBuilder<> &IRB);
231 Value *getStackBaseTag(IRBuilder<> &IRB);
232 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
233 Value *getUARTag(IRBuilder<> &IRB);
234
235 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB);
236 Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
237 unsigned retagMask(unsigned AllocaNo);
238
239 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
240
241 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
242 void instrumentGlobals();
243
244 Value *getCachedFP(IRBuilder<> &IRB);
245 Value *getFrameRecordInfo(IRBuilder<> &IRB);
246
247 void instrumentPersonalityFunctions();
248
249 const InstrumentationOptions &Opts;
250 LLVMContext *C;
251 Module &M;
252 const StackSafetyGlobalInfo *SSI;
253 Triple TargetTriple;
254 std::unique_ptr<RandomNumberGenerator> Rng;
255
256 /// This struct defines the shadow mapping using the rule:
257 /// If `kFixed`, then
258 /// shadow = (mem >> Scale) + Offset.
259 /// If `kGlobal`, then
260 /// extern char* __hwasan_shadow_memory_dynamic_address;
261 /// shadow = (mem >> Scale) + __hwasan_shadow_memory_dynamic_address
262 /// If `kIfunc`, then
263 /// extern char __hwasan_shadow[];
264 /// shadow = (mem >> Scale) + &__hwasan_shadow
265 /// If `kTls`, then
266 /// extern char *__hwasan_tls;
267 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
268 ///
269 /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
270 /// ring buffer for storing stack allocations on targets that support it.
271 class ShadowMapping {
272 OffsetKind Kind;
273 uint64_t Offset;
274 uint8_t Scale;
275 bool WithFrameRecord;
276
277 void SetFixed(uint64_t O) {
278 Kind = OffsetKind::kFixed;
279 Offset = O;
280 }
281
282 public:
283 void init(const InstrumentationOptions &Opts, Triple &TargetTriple,
284 bool InstrumentWithCalls, bool CompileKernel);
285 Align getObjectAlignment() const { return Align::fromLog2(Log2Value: Scale); }
286 bool isInGlobal() const { return Kind == OffsetKind::kGlobal; }
287 bool isInIfunc() const { return Kind == OffsetKind::kIfunc; }
288 bool isInTls() const { return Kind == OffsetKind::kTls; }
289 bool isFixed() const { return Kind == OffsetKind::kFixed; }
290 uint8_t scale() const { return Scale; };
291 uint64_t offset() const {
292 assert(isFixed());
293 return Offset;
294 };
295 bool withFrameRecord() const { return WithFrameRecord; };
296 };
297
298 ShadowMapping Mapping;
299
300 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
301 Type *IntptrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
302 PointerType *PtrTy = PointerType::getUnqual(C&: M.getContext());
303 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
304 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
305 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
306
307 bool CompileKernel;
308 bool Recover;
309 bool OutlinedChecks;
310 bool InlineFastPath;
311 bool UseShortGranules;
312 bool InstrumentLandingPads;
313 bool InstrumentWithCalls;
314 bool InstrumentStack;
315 bool InstrumentGlobals;
316 bool DetectUseAfterScope;
317 bool UsePageAliases;
318 bool UseMatchAllCallback;
319
320 std::optional<uint8_t> MatchAllTag;
321
322 unsigned PointerTagShift;
323 uint64_t TagMaskByte;
324
325 Function *HwasanCtorFunction;
326
327 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
328 FunctionCallee HwasanMemoryAccessCallbackSized[2];
329
330 FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
331 FunctionCallee HwasanHandleVfork;
332
333 FunctionCallee HwasanTagMemoryFunc;
334 FunctionCallee HwasanGenerateTagFunc;
335 FunctionCallee HwasanRecordFrameRecordFunc;
336
337 Constant *ShadowGlobal;
338
339 Value *ShadowBase = nullptr;
340 Value *StackBaseTag = nullptr;
341 Value *CachedFP = nullptr;
342 GlobalValue *ThreadPtrGlobal = nullptr;
343};
344
345} // end anonymous namespace
346
347PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
348 ModuleAnalysisManager &MAM) {
349 // Return early if nosanitize_hwaddress module flag is present for the module.
350 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_hwaddress"))
351 return PreservedAnalyses::all();
352 const StackSafetyGlobalInfo *SSI = nullptr;
353 const Triple &TargetTriple = M.getTargetTriple();
354 const InstrumentationOptions &Opts =
355 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
356 if (shouldUseStackSafetyAnalysis(Opts, TargetTriple,
357 DisableOptimization: Options.DisableOptimization))
358 SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M);
359
360 HWAddressSanitizer HWASan(Opts, M, Options.CompileKernel, Options.Recover,
361 SSI);
362 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
363 for (Function &F : M)
364 HWASan.sanitizeFunction(F, FAM);
365
366 PreservedAnalyses PA = PreservedAnalyses::none();
367 // DominatorTreeAnalysis, PostDominatorTreeAnalysis, and LoopAnalysis
368 // are incrementally updated throughout this pass whenever
369 // SplitBlockAndInsertIfThen is called.
370 PA.preserve<DominatorTreeAnalysis>();
371 PA.preserve<PostDominatorTreeAnalysis>();
372 PA.preserve<LoopAnalysis>();
373 // GlobalsAA is considered stateless and does not get invalidated unless
374 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
375 // make changes that require GlobalsAA to be invalidated.
376 PA.abandon<GlobalsAA>();
377 return PA;
378}
379void HWAddressSanitizerPass::printPipeline(
380 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
381 static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
382 OS, MapClassName2PassName);
383 OS << '<';
384 if (Options.CompileKernel)
385 OS << "kernel;";
386 if (Options.Recover)
387 OS << "recover";
388 OS << '>';
389}
390
391void HWAddressSanitizer::createHwasanNote() {
392 // Create a note that contains pointers to the list of global
393 // descriptors. Adding a note to the output file will cause the linker to
394 // create a PT_NOTE program header pointing to the note that we can use to
395 // find the descriptor list starting from the program headers. A function
396 // provided by the runtime initializes the shadow memory for the globals by
397 // accessing the descriptor list via the note. The dynamic loader needs to
398 // call this function whenever a library is loaded.
399 //
400 // The reason why we use a note for this instead of a more conventional
401 // approach of having a global constructor pass a descriptor list pointer to
402 // the runtime is because of an order of initialization problem. With
403 // constructors we can encounter the following problematic scenario:
404 //
405 // 1) library A depends on library B and also interposes one of B's symbols
406 // 2) B's constructors are called before A's (as required for correctness)
407 // 3) during construction, B accesses one of its "own" globals (actually
408 // interposed by A) and triggers a HWASAN failure due to the initialization
409 // for A not having happened yet
410 //
411 // Even without interposition it is possible to run into similar situations in
412 // cases where two libraries mutually depend on each other.
413 //
414 // We only need one note per binary, so put everything for the note in a
415 // comdat. This needs to be a comdat with an .init_array section to prevent
416 // newer versions of lld from discarding the note.
417 //
418 // Create the note even if we aren't instrumenting globals. This ensures that
419 // binaries linked from object files with both instrumented and
420 // non-instrumented globals will end up with a note, even if a comdat from an
421 // object file with non-instrumented globals is selected. The note is harmless
422 // if the runtime doesn't support it, since it will just be ignored.
423 Comdat *NoteComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
424
425 Type *Int8Arr0Ty = ArrayType::get(ElementType: Int8Ty, NumElements: 0);
426 auto *Start =
427 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
428 nullptr, "__start_hwasan_globals");
429 Start->setVisibility(GlobalValue::HiddenVisibility);
430 auto *Stop =
431 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
432 nullptr, "__stop_hwasan_globals");
433 Stop->setVisibility(GlobalValue::HiddenVisibility);
434
435 // Null-terminated so actually 8 bytes, which are required in order to align
436 // the note properly.
437 auto *Name = ConstantDataArray::get(Context&: *C, Elts: "LLVM\0\0\0");
438
439 auto *NoteTy = StructType::get(elt1: Int32Ty, elts: Int32Ty, elts: Int32Ty, elts: Name->getType(),
440 elts: Int32Ty, elts: Int32Ty);
441 auto *Note =
442 new GlobalVariable(M, NoteTy, /*isConstant=*/true,
443 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
444 Note->setSection(".note.hwasan.globals");
445 Note->setComdat(NoteComdat);
446 Note->setAlignment(Align(4));
447
448 // The pointers in the note need to be relative so that the note ends up being
449 // placed in rodata, which is the standard location for notes.
450 auto CreateRelPtr = [&](Constant *Ptr) {
451 return ConstantExpr::getTrunc(
452 C: ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: Ptr, Ty: Int64Ty),
453 C2: ConstantExpr::getPtrToInt(C: Note, Ty: Int64Ty)),
454 Ty: Int32Ty);
455 };
456 Note->setInitializer(ConstantStruct::getAnon(
457 V: {ConstantInt::get(Ty: Int32Ty, V: 8), // n_namesz
458 ConstantInt::get(Ty: Int32Ty, V: 8), // n_descsz
459 ConstantInt::get(Ty: Int32Ty, V: ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
460 Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
461 appendToCompilerUsed(M, Values: Note);
462
463 // Create a zero-length global in hwasan_globals so that the linker will
464 // always create start and stop symbols.
465 auto *Dummy = new GlobalVariable(
466 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
467 Constant::getNullValue(Ty: Int8Arr0Ty), "hwasan.dummy.global");
468 Dummy->setSection("hwasan_globals");
469 Dummy->setComdat(NoteComdat);
470 Dummy->setMetadata(KindID: LLVMContext::MD_associated,
471 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: Note)));
472 appendToCompilerUsed(M, Values: Dummy);
473}
474
475void HWAddressSanitizer::createHwasanCtorComdat() {
476 std::tie(args&: HwasanCtorFunction, args: std::ignore) =
477 getOrCreateSanitizerCtorAndInitFunctions(
478 M, CtorName: kHwasanModuleCtorName, InitName: kHwasanInitName,
479 /*InitArgTypes=*/{},
480 /*InitArgs=*/{},
481 // This callback is invoked when the functions are created the first
482 // time. Hook them into the global ctors list in that case:
483 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
484 Comdat *CtorComdat = M.getOrInsertComdat(Name: kHwasanModuleCtorName);
485 Ctor->setComdat(CtorComdat);
486 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
487 });
488
489 // Do not create .note.hwasan.globals for static binaries, as it is only
490 // needed for instrumenting globals from dynamic libraries. In static
491 // binaries, the global variables section can be accessed directly via the
492 // __start_hwasan_globals and __stop_hwasan_globals symbols inserted by the
493 // linker.
494 if (!Opts.hwasan_static_linking)
495 createHwasanNote();
496}
497
498/// Module-level initialization.
499///
500/// inserts a call to __hwasan_init to the module's constructor list.
501void HWAddressSanitizer::initializeModule() {
502 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
503 TargetTriple = M.getTargetTriple();
504
505 // HWASan may do short granule checks on function arguments read from the
506 // argument memory (last byte of the granule), which invalidates writeonly.
507 for (Function &F : M.functions())
508 removeASanIncompatibleFnAttributes(F, /*ReadsArgMem=*/true);
509
510 // x86_64 currently has two modes:
511 // - Intel LAM (default)
512 // - pointer aliasing (heap only)
513 bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
514 UsePageAliases = shouldUsePageAliases(Opts, TargetTriple);
515 InstrumentWithCalls = shouldInstrumentWithCalls(Opts, TargetTriple);
516 InstrumentStack = shouldInstrumentStack(Opts, TargetTriple);
517 DetectUseAfterScope = shouldDetectUseAfterScope(Opts, TargetTriple);
518 PointerTagShift = IsX86_64 ? 57 : 56;
519 TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
520 if (Opts.hwasan_tag_bits) {
521 if (TagMaskByte < 4)
522 reportFatalUsageError(
523 reason: "need more than 4 bits of tag to have non-short-granule tags");
524 TagMaskByte &= (1ULL << Opts.hwasan_tag_bits) - 1;
525 }
526
527 Mapping.init(Opts, TargetTriple, InstrumentWithCalls, CompileKernel);
528
529 C = &(M.getContext());
530 IRBuilder<> IRB(M);
531
532 HwasanCtorFunction = nullptr;
533
534 // Older versions of Android do not have the required runtime support for
535 // short granules, global or personality function instrumentation. On other
536 // platforms we currently require using the latest version of the runtime.
537 bool NewRuntime =
538 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(Major: 30);
539
540 UseShortGranules = valueOr(X: Opts.hwasan_use_short_granules, Default: NewRuntime);
541 OutlinedChecks = (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
542 TargetTriple.isOSBinFormatELF() &&
543 !valueOr(X: Opts.hwasan_inline_all_checks, Default: Recover);
544
545 // These platforms may prefer less inlining to reduce binary size.
546 InlineFastPath =
547 valueOr(X: Opts.hwasan_inline_fast_path_checks,
548 Default: !(TargetTriple.isAndroid() || TargetTriple.isOSFuchsia()));
549
550 if (Opts.hwasan_match_all_tag) {
551 if (*Opts.hwasan_match_all_tag != -1) {
552 MatchAllTag = *Opts.hwasan_match_all_tag & 0xFF;
553 }
554 } else if (CompileKernel) {
555 MatchAllTag = 0xFF;
556 }
557 UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
558
559 // If we don't have personality function support, fall back to landing pads.
560 InstrumentLandingPads =
561 valueOr(X: Opts.hwasan_instrument_landing_pads, Default: !NewRuntime);
562
563 InstrumentGlobals = !CompileKernel && !UsePageAliases &&
564 valueOr(X: Opts.hwasan_globals, Default: NewRuntime);
565
566 if (!CompileKernel) {
567 if (InstrumentGlobals)
568 instrumentGlobals();
569
570 createHwasanCtorComdat();
571
572 bool InstrumentPersonalityFunctions =
573 valueOr(X: Opts.hwasan_instrument_personality_functions, Default: NewRuntime);
574 if (InstrumentPersonalityFunctions)
575 instrumentPersonalityFunctions();
576 }
577
578 if (!TargetTriple.isAndroid()) {
579 ThreadPtrGlobal = M.getOrInsertGlobal(Name: "__hwasan_tls", Ty: IntptrTy, CreateGlobalCallback: [&] {
580 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
581 GlobalValue::ExternalLinkage, nullptr,
582 "__hwasan_tls", nullptr,
583 GlobalVariable::InitialExecTLSModel);
584 appendToCompilerUsed(M, Values: GV);
585 return GV;
586 });
587 }
588}
589
590void HWAddressSanitizer::initializeCallbacks(Module &M) {
591 IRBuilder<> IRB(M);
592 const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
593 FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
594 *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
595 *HwasanMemsetFnTy;
596 if (UseMatchAllCallback) {
597 HwasanMemoryAccessCallbackSizedFnTy =
598 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy, Int8Ty}, isVarArg: false);
599 HwasanMemoryAccessCallbackFnTy =
600 FunctionType::get(Result: VoidTy, Params: {IntptrTy, Int8Ty}, isVarArg: false);
601 HwasanMemTransferFnTy =
602 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy, Int8Ty}, isVarArg: false);
603 HwasanMemsetFnTy =
604 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy, Int8Ty}, isVarArg: false);
605 } else {
606 HwasanMemoryAccessCallbackSizedFnTy =
607 FunctionType::get(Result: VoidTy, Params: {IntptrTy, IntptrTy}, isVarArg: false);
608 HwasanMemoryAccessCallbackFnTy =
609 FunctionType::get(Result: VoidTy, Params: {IntptrTy}, isVarArg: false);
610 HwasanMemTransferFnTy =
611 FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, IntptrTy}, isVarArg: false);
612 HwasanMemsetFnTy =
613 FunctionType::get(Result: PtrTy, Params: {PtrTy, Int32Ty, IntptrTy}, isVarArg: false);
614 }
615
616 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
617 const std::string TypeStr = AccessIsWrite ? "store" : "load";
618 const std::string EndingStr = Recover ? "_noabort" : "";
619
620 HwasanMemoryAccessCallbackSized[AccessIsWrite] =
621 M.getOrInsertFunction(Name: (Opts.hwasan_memory_access_callback_prefix +
622 TypeStr + "N" + MatchAllStr + EndingStr)
623 .str(),
624 T: HwasanMemoryAccessCallbackSizedFnTy);
625
626 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
627 AccessSizeIndex++) {
628 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
629 M.getOrInsertFunction(Name: (Opts.hwasan_memory_access_callback_prefix +
630 TypeStr + itostr(X: 1ULL << AccessSizeIndex) +
631 MatchAllStr + EndingStr)
632 .str(),
633 T: HwasanMemoryAccessCallbackFnTy);
634 }
635 }
636
637 const std::string MemIntrinCallbackPrefix =
638 (CompileKernel && !Opts.hwasan_kernel_mem_intrinsic_prefix)
639 ? std::string("")
640 : Opts.hwasan_memory_access_callback_prefix.str();
641
642 HwasanMemmove = M.getOrInsertFunction(
643 Name: MemIntrinCallbackPrefix + "memmove" + MatchAllStr, T: HwasanMemTransferFnTy);
644 HwasanMemcpy = M.getOrInsertFunction(
645 Name: MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, T: HwasanMemTransferFnTy);
646 HwasanMemset = M.getOrInsertFunction(
647 Name: MemIntrinCallbackPrefix + "memset" + MatchAllStr, T: HwasanMemsetFnTy);
648
649 HwasanTagMemoryFunc = M.getOrInsertFunction(Name: "__hwasan_tag_memory", RetTy: VoidTy,
650 Args: PtrTy, Args: Int8Ty, Args: IntptrTy);
651 HwasanGenerateTagFunc =
652 M.getOrInsertFunction(Name: "__hwasan_generate_tag", RetTy: Int8Ty);
653
654 HwasanRecordFrameRecordFunc =
655 M.getOrInsertFunction(Name: "__hwasan_add_frame_record", RetTy: VoidTy, Args: Int64Ty);
656
657 ShadowGlobal =
658 M.getOrInsertGlobal(Name: "__hwasan_shadow", Ty: ArrayType::get(ElementType: Int8Ty, NumElements: 0));
659
660 HwasanHandleVfork =
661 M.getOrInsertFunction(Name: "__hwasan_handle_vfork", RetTy: VoidTy, Args: IntptrTy);
662}
663
664Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
665 // An empty inline asm with input reg == output reg.
666 // An opaque no-op cast, basically.
667 // This prevents code bloat as a result of rematerializing trivial definitions
668 // such as constants or global addresses at every load and store.
669 InlineAsm *Asm =
670 InlineAsm::get(Ty: FunctionType::get(Result: PtrTy, Params: {Val->getType()}, isVarArg: false),
671 AsmString: StringRef(""), Constraints: StringRef("=r,0"),
672 /*hasSideEffects=*/false);
673 return IRB.CreateCall(Callee: Asm, Args: {Val}, Name: ".hwasan.shadow");
674}
675
676Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
677 return getOpaqueNoopCast(IRB, Val: ShadowGlobal);
678}
679
680Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
681 if (Mapping.isFixed()) {
682 return getOpaqueNoopCast(
683 IRB, Val: ConstantExpr::getIntToPtr(
684 C: ConstantInt::get(Ty: IntptrTy, V: Mapping.offset()), Ty: PtrTy));
685 }
686
687 if (Mapping.isInIfunc())
688 return getDynamicShadowIfunc(IRB);
689
690 Value *GlobalDynamicAddress = IRB.getModule()->getOrInsertGlobal(
691 Name: kHwasanShadowMemoryDynamicAddress, Ty: PtrTy);
692 return IRB.CreateLoad(Ty: PtrTy, Ptr: GlobalDynamicAddress);
693}
694
695bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst,
696 Value *Ptr) {
697 // Do not instrument accesses from different address spaces; we cannot deal
698 // with them.
699 Type *PtrTy = cast<PointerType>(Val: Ptr->getType()->getScalarType());
700 if (PtrTy->getPointerAddressSpace() != 0)
701 return true;
702
703 // Ignore swifterror addresses.
704 // swifterror memory addresses are mem2reg promoted by instruction
705 // selection. As such they cannot have regular uses like an instrumentation
706 // function and it makes no sense to track them as memory.
707 if (Ptr->isSwiftError())
708 return true;
709
710 if (findAllocaForValue(V: Ptr)) {
711 if (!InstrumentStack)
712 return true;
713 if (SSI && SSI->stackAccessIsSafe(I: *Inst))
714 return true;
715 }
716
717 if (isa<GlobalVariable>(Val: getUnderlyingObject(V: Ptr))) {
718 if (!InstrumentGlobals)
719 return true;
720 // TODO: Optimize inbound global accesses, like Asan `instrumentMop`.
721 }
722
723 return false;
724}
725
726bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE,
727 Instruction *Inst, Value *Ptr) {
728 bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr);
729 if (Ignored) {
730 ORE.emit(
731 RemarkBuilder: [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); });
732 } else {
733 ORE.emit(RemarkBuilder: [&]() {
734 return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst);
735 });
736 }
737 return Ignored;
738}
739
740void HWAddressSanitizer::getInterestingMemoryOperands(
741 OptimizationRemarkEmitter &ORE, Instruction *I,
742 const TargetLibraryInfo &TLI,
743 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
744 // Skip memory accesses inserted by another instrumentation.
745 if (I->hasMetadata(KindID: LLVMContext::MD_nosanitize))
746 return;
747
748 // Do not instrument the load fetching the dynamic shadow address.
749 if (ShadowBase == I)
750 return;
751
752 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
753 if (!Opts.hwasan_instrument_reads ||
754 ignoreAccess(ORE, Inst: I, Ptr: LI->getPointerOperand()))
755 return;
756 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
757 Args: LI->getType(), Args: LI->getAlign());
758 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
759 if (!Opts.hwasan_instrument_writes ||
760 ignoreAccess(ORE, Inst: I, Ptr: SI->getPointerOperand()))
761 return;
762 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
763 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
764 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
765 if (!Opts.hwasan_instrument_atomics ||
766 ignoreAccess(ORE, Inst: I, Ptr: RMW->getPointerOperand()))
767 return;
768 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
769 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
770 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
771 if (!Opts.hwasan_instrument_atomics ||
772 ignoreAccess(ORE, Inst: I, Ptr: XCHG->getPointerOperand()))
773 return;
774 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
775 Args: XCHG->getCompareOperand()->getType(),
776 Args: std::nullopt);
777 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
778 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
779 if (!Opts.hwasan_instrument_byval || !CI->isByValArgument(ArgNo) ||
780 ignoreAccess(ORE, Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
781 continue;
782 Type *Ty = CI->getParamByValType(ArgNo);
783 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
784 }
785 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
786 }
787}
788
789static unsigned getPointerOperandIndex(Instruction *I) {
790 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I))
791 return LI->getPointerOperandIndex();
792 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I))
793 return SI->getPointerOperandIndex();
794 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I))
795 return RMW->getPointerOperandIndex();
796 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I))
797 return XCHG->getPointerOperandIndex();
798 report_fatal_error(reason: "Unexpected instruction");
799 return -1;
800}
801
802static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
803 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
804 assert(Res < kNumberOfAccessSizes);
805 return Res;
806}
807
808void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
809 if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
810 TargetTriple.isRISCV64())
811 return;
812
813 IRBuilder<> IRB(I);
814 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
815 Value *UntaggedPtr =
816 IRB.CreateIntToPtr(V: untagPointer(IRB, PtrLong: AddrLong), DestTy: Addr->getType());
817 I->setOperand(i: getPointerOperandIndex(I), Val: UntaggedPtr);
818}
819
820Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
821 // Mem >> Scale
822 Value *Shadow = IRB.CreateLShr(LHS: Mem, RHS: Mapping.scale());
823 if (Mapping.isFixed() && Mapping.offset() == 0)
824 return IRB.CreateIntToPtr(V: Shadow, DestTy: PtrTy);
825 // (Mem >> Scale) + Offset
826 return IRB.CreatePtrAdd(Ptr: ShadowBase, Offset: Shadow);
827}
828
829int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
830 unsigned AccessSizeIndex) {
831 return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
832 (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
833 (MatchAllTag.value_or(u: 0) << HWASanAccessInfo::MatchAllShift) |
834 (Recover << HWASanAccessInfo::RecoverShift) |
835 (IsWrite << HWASanAccessInfo::IsWriteShift) |
836 (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
837}
838
839HWAddressSanitizer::ShadowTagCheckInfo
840HWAddressSanitizer::insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
841 DomTreeUpdater &DTU, LoopInfo *LI) {
842 ShadowTagCheckInfo R;
843
844 IRBuilder<> IRB(InsertBefore);
845
846 R.PtrLong = IRB.CreatePointerCast(V: Ptr, DestTy: IntptrTy);
847 R.PtrTag =
848 IRB.CreateTrunc(V: IRB.CreateLShr(LHS: R.PtrLong, RHS: PointerTagShift), DestTy: Int8Ty);
849 R.AddrLong = untagPointer(IRB, PtrLong: R.PtrLong);
850 Value *Shadow = memToShadow(Mem: R.AddrLong, IRB);
851 R.MemTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: Shadow);
852 Value *TagMismatch = IRB.CreateICmpNE(LHS: R.PtrTag, RHS: R.MemTag);
853
854 if (MatchAllTag.has_value()) {
855 Value *TagNotIgnored = IRB.CreateICmpNE(
856 LHS: R.PtrTag, RHS: ConstantInt::get(Ty: R.PtrTag->getType(), V: *MatchAllTag));
857 TagMismatch = IRB.CreateAnd(LHS: TagMismatch, RHS: TagNotIgnored);
858 }
859
860 R.TagMismatchTerm = SplitBlockAndInsertIfThen(
861 Cond: TagMismatch, SplitBefore: InsertBefore, Unreachable: false,
862 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
863
864 return R;
865}
866
867void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
868 unsigned AccessSizeIndex,
869 Instruction *InsertBefore,
870 DomTreeUpdater &DTU,
871 LoopInfo *LI) {
872 assert(!UsePageAliases);
873 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
874
875 if (InlineFastPath)
876 InsertBefore =
877 insertShadowTagCheck(Ptr, InsertBefore, DTU, LI).TagMismatchTerm;
878
879 IRBuilder<> IRB(InsertBefore);
880 bool UseFixedShadowIntrinsic = false;
881 // The memaccess fixed shadow intrinsic is only supported on AArch64,
882 // which allows a 16-bit immediate to be left-shifted by 32.
883 // Since kShadowBaseAlignment == 32, and Linux by default will not
884 // mmap above 48-bits, practically any valid shadow offset is
885 // representable.
886 // In particular, an offset of 4TB (1024 << 32) is representable, and
887 // ought to be good enough for anybody.
888 if (TargetTriple.isAArch64() && Mapping.isFixed()) {
889 uint16_t OffsetShifted = Mapping.offset() >> 32;
890 UseFixedShadowIntrinsic =
891 static_cast<uint64_t>(OffsetShifted) << 32 == Mapping.offset();
892 }
893
894 if (UseFixedShadowIntrinsic) {
895 IRB.CreateIntrinsic(
896 ID: UseShortGranules
897 ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
898 : Intrinsic::hwasan_check_memaccess_fixedshadow,
899 Args: {Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo),
900 ConstantInt::get(Ty: Int64Ty, V: Mapping.offset())});
901 } else {
902 IRB.CreateIntrinsic(
903 ID: UseShortGranules ? Intrinsic::hwasan_check_memaccess_shortgranules
904 : Intrinsic::hwasan_check_memaccess,
905 Args: {ShadowBase, Ptr, ConstantInt::get(Ty: Int32Ty, V: AccessInfo)});
906 }
907}
908
909void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
910 unsigned AccessSizeIndex,
911 Instruction *InsertBefore,
912 DomTreeUpdater &DTU,
913 LoopInfo *LI) {
914 assert(!UsePageAliases);
915 const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
916
917 ShadowTagCheckInfo TCI = insertShadowTagCheck(Ptr, InsertBefore, DTU, LI);
918
919 IRBuilder<> IRB(TCI.TagMismatchTerm);
920 Value *OutOfShortGranuleTagRange =
921 IRB.CreateICmpUGT(LHS: TCI.MemTag, RHS: ConstantInt::get(Ty: Int8Ty, V: 15));
922 Instruction *CheckFailTerm = SplitBlockAndInsertIfThen(
923 Cond: OutOfShortGranuleTagRange, SplitBefore: TCI.TagMismatchTerm, Unreachable: !Recover,
924 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU, LI);
925
926 IRB.SetInsertPoint(TCI.TagMismatchTerm);
927 Value *PtrLowBits = IRB.CreateTrunc(V: IRB.CreateAnd(LHS: TCI.PtrLong, RHS: 15), DestTy: Int8Ty);
928 PtrLowBits = IRB.CreateAdd(
929 LHS: PtrLowBits, RHS: ConstantInt::get(Ty: Int8Ty, V: (1 << AccessSizeIndex) - 1));
930 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(LHS: PtrLowBits, RHS: TCI.MemTag);
931 SplitBlockAndInsertIfThen(Cond: PtrLowBitsOOB, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
932 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
933 LI, ThenBlock: CheckFailTerm->getParent());
934
935 IRB.SetInsertPoint(TCI.TagMismatchTerm);
936 Value *InlineTagAddr = IRB.CreateOr(LHS: TCI.AddrLong, RHS: 15);
937 InlineTagAddr = IRB.CreateIntToPtr(V: InlineTagAddr, DestTy: PtrTy);
938 Value *InlineTag = IRB.CreateLoad(Ty: Int8Ty, Ptr: InlineTagAddr);
939 Value *InlineTagMismatch = IRB.CreateICmpNE(LHS: TCI.PtrTag, RHS: InlineTag);
940 SplitBlockAndInsertIfThen(Cond: InlineTagMismatch, SplitBefore: TCI.TagMismatchTerm, Unreachable: false,
941 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights(), DTU: &DTU,
942 LI, ThenBlock: CheckFailTerm->getParent());
943
944 IRB.SetInsertPoint(CheckFailTerm);
945 InlineAsm *Asm;
946 switch (TargetTriple.getArch()) {
947 case Triple::x86_64:
948 // The signal handler will find the data address in rdi.
949 Asm = InlineAsm::get(
950 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
951 AsmString: "int3\nnopl " +
952 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
953 "(%rax)",
954 Constraints: "{rdi}",
955 /*hasSideEffects=*/true);
956 break;
957 case Triple::aarch64:
958 case Triple::aarch64_be:
959 // The signal handler will find the data address in x0.
960 Asm = InlineAsm::get(
961 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
962 AsmString: "brk #" + itostr(X: 0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
963 Constraints: "{x0}",
964 /*hasSideEffects=*/true);
965 break;
966 case Triple::riscv64:
967 // The signal handler will find the data address in x10.
968 Asm = InlineAsm::get(
969 Ty: FunctionType::get(Result: VoidTy, Params: {TCI.PtrLong->getType()}, isVarArg: false),
970 AsmString: "ebreak\naddiw x0, x11, " +
971 itostr(X: 0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
972 Constraints: "{x10}",
973 /*hasSideEffects=*/true);
974 break;
975 default:
976 report_fatal_error(reason: "unsupported architecture");
977 }
978 IRB.CreateCall(Callee: Asm, Args: TCI.PtrLong);
979 if (Recover)
980 cast<UncondBrInst>(Val: CheckFailTerm)
981 ->setSuccessor(TCI.TagMismatchTerm->getParent());
982}
983
984bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE,
985 MemIntrinsic *MI) {
986 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(Val: MI)) {
987 return (!Opts.hwasan_instrument_writes ||
988 ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getDest())) &&
989 (!Opts.hwasan_instrument_reads ||
990 ignoreAccess(ORE, Inst: MTI, Ptr: MTI->getSource()));
991 }
992 if (isa<MemSetInst>(Val: MI))
993 return !Opts.hwasan_instrument_writes ||
994 ignoreAccess(ORE, Inst: MI, Ptr: MI->getDest());
995 return false;
996}
997
998void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
999 IRBuilder<> IRB(MI);
1000 if (isa<MemTransferInst>(Val: MI)) {
1001 SmallVector<Value *, 4> Args{
1002 MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
1003 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1004
1005 if (UseMatchAllCallback)
1006 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1007 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? HwasanMemmove : HwasanMemcpy, Args);
1008 } else if (isa<MemSetInst>(Val: MI)) {
1009 SmallVector<Value *, 4> Args{
1010 MI->getOperand(i_nocapture: 0),
1011 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1012 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)};
1013 if (UseMatchAllCallback)
1014 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1015 IRB.CreateCall(Callee: HwasanMemset, Args);
1016 }
1017 MI->eraseFromParent();
1018}
1019
1020bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O,
1021 DomTreeUpdater &DTU, LoopInfo *LI,
1022 const DataLayout &DL) {
1023 Value *Addr = O.getPtr();
1024
1025 LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
1026
1027 // If the pointer is statically known to be zero, the tag check will pass
1028 // since:
1029 // 1) it has a zero tag
1030 // 2) the shadow memory corresponding to address 0 is initialized to zero and
1031 // never updated.
1032 // We can therefore elide the tag check.
1033 llvm::KnownBits Known(DL.getPointerTypeSizeInBits(Addr->getType()));
1034 llvm::computeKnownBits(V: Addr, Known, DL);
1035 if (Known.isZero())
1036 return false;
1037
1038 if (O.MaybeMask)
1039 return false; // FIXME
1040
1041 IRBuilder<> IRB(O.getInsn());
1042 if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(Value: O.TypeStoreSize) &&
1043 (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1044 (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
1045 *O.Alignment >= O.TypeStoreSize / 8)) {
1046 size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize: O.TypeStoreSize);
1047 if (InstrumentWithCalls) {
1048 SmallVector<Value *, 2> Args{IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy)};
1049 if (UseMatchAllCallback)
1050 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1051 IRB.CreateCall(Callee: HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1052 Args);
1053 } else if (OutlinedChecks) {
1054 instrumentMemAccessOutline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1055 DTU, LI);
1056 } else {
1057 instrumentMemAccessInline(Ptr: Addr, IsWrite: O.IsWrite, AccessSizeIndex, InsertBefore: O.getInsn(),
1058 DTU, LI);
1059 }
1060 } else {
1061 SmallVector<Value *, 3> Args{
1062 IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy),
1063 IRB.CreateUDiv(LHS: IRB.CreateTypeSize(Ty: IntptrTy, Size: O.TypeStoreSize),
1064 RHS: ConstantInt::get(Ty: IntptrTy, V: 8))};
1065 if (UseMatchAllCallback)
1066 Args.emplace_back(Args: ConstantInt::get(Ty: Int8Ty, V: *MatchAllTag));
1067 IRB.CreateCall(Callee: HwasanMemoryAccessCallbackSized[O.IsWrite], Args);
1068 }
1069 untagPointerOperand(I: O.getInsn(), Addr);
1070
1071 return true;
1072}
1073
1074void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1075 size_t Size) {
1076 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1077 if (!UseShortGranules)
1078 Size = AlignedSize;
1079
1080 Tag = IRB.CreateTrunc(V: Tag, DestTy: Int8Ty);
1081 if (InstrumentWithCalls) {
1082 IRB.CreateCall(Callee: HwasanTagMemoryFunc,
1083 Args: {IRB.CreatePointerCast(V: AI, DestTy: PtrTy), Tag,
1084 ConstantInt::get(Ty: IntptrTy, V: AlignedSize)});
1085 } else {
1086 size_t ShadowSize = Size >> Mapping.scale();
1087 Value *AddrLong = untagPointer(IRB, PtrLong: IRB.CreatePointerCast(V: AI, DestTy: IntptrTy));
1088 Value *ShadowPtr = memToShadow(Mem: AddrLong, IRB);
1089 // If this memset is not inlined, it will be intercepted in the hwasan
1090 // runtime library. That's OK, because the interceptor skips the checks if
1091 // the address is in the shadow region.
1092 // FIXME: the interceptor is not as fast as real memset. Consider lowering
1093 // llvm.memset right here into either a sequence of stores, or a call to
1094 // hwasan_tag_memory.
1095 if (ShadowSize)
1096 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Tag, Size: ShadowSize, Align: Align(1));
1097 if (Size != AlignedSize) {
1098 const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1099 IRB.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: SizeRemainder),
1100 Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: ShadowPtr, Idx0: ShadowSize));
1101 IRB.CreateStore(
1102 Val: Tag, Ptr: IRB.CreateConstGEP1_32(Ty: Int8Ty, Ptr: IRB.CreatePointerCast(V: AI, DestTy: PtrTy),
1103 Idx0: AlignedSize - 1));
1104 }
1105 }
1106}
1107
1108unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1109 if (TargetTriple.getArch() == Triple::x86_64)
1110 return AllocaNo & TagMaskByte;
1111
1112 // A list of 8-bit numbers that have at most one run of non-zero bits.
1113 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1114 // masks.
1115 // The list does not include the value 255, which is used for UAR.
1116 //
1117 // Because we are more likely to use earlier elements of this list than later
1118 // ones, it is sorted in increasing order of probability of collision with a
1119 // mask allocated (temporally) nearby. The program that generated this list
1120 // can be found at:
1121 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1122 static const unsigned FastMasks[] = {
1123 0, 128, 64, 192, 32, 96, 224, 112, 240, 48, 16, 120,
1124 248, 56, 24, 8, 124, 252, 60, 28, 12, 4, 126, 254,
1125 62, 30, 14, 6, 2, 127, 63, 31, 15, 7, 3, 1};
1126 return FastMasks[AllocaNo % std::size(FastMasks)];
1127}
1128
1129Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1130 if (TagMaskByte == 0xFF)
1131 return OldTag; // No need to clear the tag byte.
1132 return IRB.CreateAnd(LHS: OldTag,
1133 RHS: ConstantInt::get(Ty: OldTag->getType(), V: TagMaskByte));
1134}
1135
1136Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1137 return IRB.CreateZExt(V: IRB.CreateCall(Callee: HwasanGenerateTagFunc), DestTy: IntptrTy);
1138}
1139
1140Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1141 if (Opts.hwasan_generate_tags_with_calls)
1142 return nullptr;
1143 if (StackBaseTag)
1144 return StackBaseTag;
1145 // Extract some entropy from the stack pointer for the tags.
1146 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1147 // between functions).
1148 Value *FramePointerLong = getCachedFP(IRB);
1149 Value *StackTag =
1150 applyTagMask(IRB, OldTag: IRB.CreateXor(LHS: FramePointerLong,
1151 RHS: IRB.CreateLShr(LHS: FramePointerLong, RHS: 20)));
1152 StackTag->setName("hwasan.stack.base.tag");
1153 return StackTag;
1154}
1155
1156Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1157 unsigned AllocaNo) {
1158 if (Opts.hwasan_generate_tags_with_calls)
1159 return getNextTagWithCall(IRB);
1160 return IRB.CreateXor(
1161 LHS: StackTag, RHS: ConstantInt::get(Ty: StackTag->getType(), V: retagMask(AllocaNo)));
1162}
1163
1164Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
1165 Value *FramePointerLong = getCachedFP(IRB);
1166 Value *UARTag =
1167 applyTagMask(IRB, OldTag: IRB.CreateLShr(LHS: FramePointerLong, RHS: PointerTagShift));
1168
1169 UARTag->setName("hwasan.uar.tag");
1170 return UARTag;
1171}
1172
1173// Add a tag to an address.
1174Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1175 Value *PtrLong, Value *Tag) {
1176 assert(!UsePageAliases);
1177 Value *TaggedPtrLong;
1178 if (CompileKernel) {
1179 // Kernel addresses have 0xFF in the most significant byte.
1180 Value *ShiftedTag =
1181 IRB.CreateOr(LHS: IRB.CreateShl(LHS: Tag, RHS: PointerTagShift),
1182 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << PointerTagShift) - 1));
1183 TaggedPtrLong = IRB.CreateAnd(LHS: PtrLong, RHS: ShiftedTag);
1184 } else {
1185 // Userspace can simply do OR (tag << PointerTagShift);
1186 Value *ShiftedTag = IRB.CreateShl(LHS: Tag, RHS: PointerTagShift);
1187 TaggedPtrLong = IRB.CreateOr(LHS: PtrLong, RHS: ShiftedTag);
1188 }
1189 return IRB.CreateIntToPtr(V: TaggedPtrLong, DestTy: Ty);
1190}
1191
1192// Remove tag from an address.
1193Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1194 assert(!UsePageAliases);
1195 Value *UntaggedPtrLong;
1196 if (CompileKernel) {
1197 // Kernel addresses have 0xFF in the most significant byte.
1198 UntaggedPtrLong =
1199 IRB.CreateOr(LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1200 V: TagMaskByte << PointerTagShift));
1201 } else {
1202 // Userspace addresses have 0x00.
1203 UntaggedPtrLong = IRB.CreateAnd(
1204 LHS: PtrLong, RHS: ConstantInt::get(Ty: PtrLong->getType(),
1205 V: ~(TagMaskByte << PointerTagShift)));
1206 }
1207 return UntaggedPtrLong;
1208}
1209
1210Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) {
1211 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1212 // in Bionic's libc/platform/bionic/tls_defines.h.
1213 constexpr int SanitizerSlot = 6;
1214 if (TargetTriple.isAArch64() && TargetTriple.isAndroid())
1215 return memtag::getAndroidSlotPtr(IRB, Slot: SanitizerSlot);
1216 return ThreadPtrGlobal;
1217}
1218
1219Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) {
1220 if (!CachedFP)
1221 CachedFP = memtag::getFP(IRB);
1222 return CachedFP;
1223}
1224
1225Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1226 // Prepare ring buffer data.
1227 Value *PC = memtag::getPC(TargetTriple, IRB);
1228 Value *FP = getCachedFP(IRB);
1229
1230 // Mix FP and PC.
1231 // Assumptions:
1232 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
1233 // FP is 0xfffffffffffFFFF0 (4 lower bits are zero)
1234 // We only really need ~20 lower non-zero bits (FFFF), so we mix like this:
1235 // 0xFFFFPPPPPPPPPPPP
1236 //
1237 // FP works because in AArch64FrameLowering::getFrameIndexReference, we
1238 // prefer FP-relative offsets for functions compiled with HWASan.
1239 FP = IRB.CreateShl(LHS: FP, RHS: 44);
1240 return IRB.CreateOr(LHS: PC, RHS: FP);
1241}
1242
1243void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1244 if (!Mapping.isInTls())
1245 ShadowBase = getShadowNonTls(IRB);
1246 else if (!WithFrameRecord && TargetTriple.isAndroid())
1247 ShadowBase = getDynamicShadowIfunc(IRB);
1248
1249 if (!WithFrameRecord && ShadowBase)
1250 return;
1251
1252 Value *SlotPtr = nullptr;
1253 Value *ThreadLong = nullptr;
1254 Value *ThreadLongMaybeUntagged = nullptr;
1255
1256 auto getThreadLongMaybeUntagged = [&]() {
1257 if (!SlotPtr)
1258 SlotPtr = getHwasanThreadSlotPtr(IRB);
1259 if (!ThreadLong)
1260 ThreadLong = IRB.CreateLoad(Ty: IntptrTy, Ptr: SlotPtr);
1261 // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1262 // TBI.
1263 return TargetTriple.isAArch64() ? ThreadLong
1264 : untagPointer(IRB, PtrLong: ThreadLong);
1265 };
1266
1267 if (WithFrameRecord) {
1268 switch (Opts.hwasan_record_stack_history) {
1269 case RecordStackHistoryMode::libcall: {
1270 // Emit a runtime call into hwasan rather than emitting instructions for
1271 // recording stack history.
1272 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1273 IRB.CreateCall(Callee: HwasanRecordFrameRecordFunc, Args: {FrameRecordInfo});
1274 break;
1275 }
1276 case RecordStackHistoryMode::instr: {
1277 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1278
1279 StackBaseTag = IRB.CreateAShr(LHS: ThreadLong, RHS: 3);
1280
1281 // Store data to ring buffer.
1282 Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1283 Value *RecordPtr =
1284 IRB.CreateIntToPtr(V: ThreadLongMaybeUntagged, DestTy: IRB.getPtrTy(AddrSpace: 0));
1285 IRB.CreateStore(Val: FrameRecordInfo, Ptr: RecordPtr);
1286
1287 IRB.CreateStore(Val: memtag::incrementThreadLong(IRB, ThreadLong, Inc: 8), Ptr: SlotPtr);
1288 break;
1289 }
1290 case RecordStackHistoryMode::none: {
1291 llvm_unreachable(
1292 "A stack history recording mode should've been selected.");
1293 }
1294 }
1295 }
1296
1297 if (!ShadowBase) {
1298 if (!ThreadLongMaybeUntagged)
1299 ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1300
1301 // Get shadow base address by aligning RecordPtr up.
1302 // Note: this is not correct if the pointer is already aligned.
1303 // Runtime library will make sure this never happens.
1304 ShadowBase = IRB.CreateAdd(
1305 LHS: IRB.CreateOr(
1306 LHS: ThreadLongMaybeUntagged,
1307 RHS: ConstantInt::get(Ty: IntptrTy, V: (1ULL << kShadowBaseAlignment) - 1)),
1308 RHS: ConstantInt::get(Ty: IntptrTy, V: 1), Name: "hwasan.shadow");
1309 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: PtrTy);
1310 }
1311}
1312
1313void HWAddressSanitizer::instrumentLandingPads(
1314 SmallVectorImpl<Instruction *> &LandingPadVec) {
1315 for (auto *LP : LandingPadVec) {
1316 IRBuilder<> IRB(LP->getNextNode());
1317 IRB.CreateCall(
1318 Callee: HwasanHandleVfork,
1319 Args: {memtag::readRegister(
1320 IRB, Name: (TargetTriple.getArch() == Triple::x86_64) ? "rsp" : "sp")});
1321 }
1322}
1323
1324void HWAddressSanitizer::instrumentStack(OptimizationRemarkEmitter &ORE,
1325 memtag::StackInfo &SInfo,
1326 Value *StackTag, Value *UARTag,
1327 const DominatorTree &DT,
1328 const PostDominatorTree &PDT,
1329 const LoopInfo &LI) {
1330 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1331 // alloca addresses using that. Unfortunately, offsets are not known yet
1332 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1333 // temp, shift-OR it into each alloca address and xor with the retag mask.
1334 // This generates one extra instruction per alloca use.
1335 unsigned int I = 0;
1336
1337 for (auto &KV : SInfo.AllocasToInstrument) {
1338 auto N = I++;
1339 auto *AI = KV.first;
1340 memtag::AllocaInfo &Info = KV.second;
1341 IRBuilder<> IRB(AI->getNextNode());
1342
1343 // Replace uses of the alloca with tagged address.
1344 Value *Tag = getAllocaTag(IRB, StackTag, AllocaNo: N);
1345 Value *AILong = IRB.CreatePointerCast(V: AI, DestTy: IntptrTy);
1346 Value *AINoTagLong = untagPointer(IRB, PtrLong: AILong);
1347 Value *Replacement = tagPointer(IRB, Ty: AI->getType(), PtrLong: AINoTagLong, Tag);
1348 std::string Name =
1349 AI->hasName() ? AI->getName().str() : "alloca." + itostr(X: N);
1350 Replacement->setName(Name + ".hwasan");
1351
1352 size_t Size = memtag::getAllocaSizeInBytes(AI: *AI);
1353 size_t AlignedSize = alignTo(Size, A: Mapping.getObjectAlignment());
1354
1355 AI->replaceUsesWithIf(New: Replacement, ShouldReplace: [AILong](const Use &U) {
1356 auto *User = U.getUser();
1357 return User != AILong && !isa<LifetimeIntrinsic>(Val: User);
1358 });
1359
1360 memtag::annotateDebugRecords(Info, Tag: retagMask(AllocaNo: N));
1361
1362 auto TagStarts = [&]() {
1363 for (IntrinsicInst *Start : Info.LifetimeStart) {
1364 IRB.SetInsertPoint(Start->getNextNode());
1365 tagAlloca(IRB, AI, Tag, Size);
1366 }
1367 };
1368 auto TagEnd = [&](Instruction *Node) {
1369 IRB.SetInsertPoint(Node);
1370 // When untagging, use the `AlignedSize` because we need to set the tags
1371 // for the entire alloca to original. If we used `Size` here, we would
1372 // keep the last granule tagged, and store zero in the last byte of the
1373 // last granule, due to how short granules are implemented.
1374 tagAlloca(IRB, AI, Tag: UARTag, Size: AlignedSize);
1375 };
1376 auto EraseLifetimes = [&]() {
1377 for (auto &II : Info.LifetimeStart)
1378 II->eraseFromParent();
1379 for (auto &II : Info.LifetimeEnd)
1380 II->eraseFromParent();
1381 };
1382 // Calls to functions that may return twice (e.g. setjmp) confuse the
1383 // postdominator analysis, and will leave us to keep memory tagged after
1384 // function return. Work around this by always untagging at every return
1385 // statement if return_twice functions are called.
1386 if (DetectUseAfterScope && !SInfo.CallsReturnTwice &&
1387 memtag::isSupportedLifetime(AInfo: Info, DT: &DT, LI: &LI)) {
1388 TagStarts();
1389 memtag::forAllReachableExits(DT, PDT, LI, AInfo: Info, RetVec: SInfo.RetVec, Callback: TagEnd);
1390 ORE.emit(RemarkBuilder: [&]() {
1391 return OptimizationRemark(DEBUG_TYPE, "supportedLifetime", AI);
1392 });
1393 } else if (DetectUseAfterScope && Opts.hwasan_strict_use_after_scope) {
1394 // SInfo.CallsReturnTwice || !isStandardLifetime
1395 ORE.emit(RemarkBuilder: [&]() {
1396 return OptimizationRemarkMissed(DEBUG_TYPE, "supportedLifetime", AI);
1397 });
1398
1399 tagAlloca(IRB, AI, Tag, Size);
1400 TagStarts();
1401 for_each(Range&: Info.LifetimeEnd, F: TagEnd);
1402 for_each(Range&: SInfo.RetVec, F: TagEnd);
1403 EraseLifetimes();
1404 } else {
1405 tagAlloca(IRB, AI, Tag, Size);
1406 for_each(Range&: SInfo.RetVec, F: TagEnd);
1407 EraseLifetimes();
1408 }
1409 memtag::alignAndPadAlloca(Info, Align: Mapping.getObjectAlignment());
1410 }
1411}
1412
1413static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE,
1414 bool Skip) {
1415 if (Skip) {
1416 ORE.emit(RemarkBuilder: [&]() {
1417 return OptimizationRemark(DEBUG_TYPE, "Skip", &F)
1418 << "Skipped: F=" << ore::NV("Function", &F);
1419 });
1420 } else {
1421 ORE.emit(RemarkBuilder: [&]() {
1422 return OptimizationRemarkMissed(DEBUG_TYPE, "Sanitize", &F)
1423 << "Sanitized: F=" << ore::NV("Function", &F);
1424 });
1425 }
1426}
1427
1428bool HWAddressSanitizer::selectiveInstrumentationShouldSkip(
1429 Function &F, FunctionAnalysisManager &FAM) const {
1430 auto SkipHot = [&]() {
1431 if (!Opts.hwasan_percentile_cutoff_hot)
1432 return false;
1433 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1434 ProfileSummaryInfo *PSI =
1435 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
1436 if (!PSI || !PSI->hasProfileSummary()) {
1437 ++NumNoProfileSummaryFuncs;
1438 return false;
1439 }
1440 return PSI->isFunctionHotInCallGraphNthPercentile(
1441 PercentileCutoff: *Opts.hwasan_percentile_cutoff_hot, F: &F,
1442 BFI&: FAM.getResult<BlockFrequencyAnalysis>(IR&: F));
1443 };
1444
1445 auto SkipRandom = [&]() {
1446 if (!Opts.hwasan_random_rate)
1447 return false;
1448 std::bernoulli_distribution D(*Opts.hwasan_random_rate);
1449 return !D(*Rng);
1450 };
1451
1452 bool Skip = SkipRandom() || SkipHot();
1453 emitRemark(F, ORE&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F), Skip);
1454 return Skip;
1455}
1456
1457void HWAddressSanitizer::sanitizeFunction(Function &F,
1458 FunctionAnalysisManager &FAM) {
1459 if (&F == HwasanCtorFunction)
1460 return;
1461
1462 // Do not apply any instrumentation for naked functions.
1463 if (F.hasFnAttribute(Kind: Attribute::Naked))
1464 return;
1465
1466 if (!F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1467 return;
1468
1469 if (F.empty())
1470 return;
1471
1472 if (F.isPresplitCoroutine())
1473 return;
1474
1475 NumTotalFuncs++;
1476
1477 OptimizationRemarkEmitter &ORE =
1478 FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
1479
1480 if (selectiveInstrumentationShouldSkip(F, FAM))
1481 return;
1482
1483 NumInstrumentedFuncs++;
1484
1485 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1486
1487 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1488 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1489 SmallVector<Instruction *, 8> LandingPadVec;
1490 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1491
1492 memtag::StackInfoBuilder SIB(SSI, DEBUG_TYPE);
1493 for (auto &Inst : instructions(F)) {
1494 if (InstrumentStack) {
1495 SIB.visit(ORE, Inst);
1496 }
1497
1498 if (InstrumentLandingPads && isa<LandingPadInst>(Val: Inst))
1499 LandingPadVec.push_back(Elt: &Inst);
1500
1501 getInterestingMemoryOperands(ORE, I: &Inst, TLI, Interesting&: OperandsToInstrument);
1502
1503 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst))
1504 if (!ignoreMemIntrinsic(ORE, MI))
1505 IntrinToInstrument.push_back(Elt: MI);
1506 }
1507
1508 memtag::StackInfo &SInfo = SIB.get();
1509
1510 initializeCallbacks(M&: *F.getParent());
1511
1512 if (!LandingPadVec.empty())
1513 instrumentLandingPads(LandingPadVec);
1514
1515 if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1516 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1517 // __hwasan_personality_thunk is a no-op for functions without an
1518 // instrumented stack, so we can drop it.
1519 F.setPersonalityFn(nullptr);
1520 }
1521
1522 if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1523 IntrinToInstrument.empty())
1524 return;
1525
1526 assert(!ShadowBase);
1527
1528 BasicBlock::iterator InsertPt = F.getEntryBlock().begin();
1529 IRBuilder<> EntryIRB(InsertPt);
1530 emitPrologue(IRB&: EntryIRB,
1531 /*WithFrameRecord*/ Opts.hwasan_record_stack_history !=
1532 RecordStackHistoryMode::none &&
1533 Mapping.withFrameRecord() &&
1534 !SInfo.AllocasToInstrument.empty());
1535
1536 if (!SInfo.AllocasToInstrument.empty()) {
1537 const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
1538 const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
1539 const LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: F);
1540 Value *StackTag = getStackBaseTag(IRB&: EntryIRB);
1541 Value *UARTag = getUARTag(IRB&: EntryIRB);
1542 instrumentStack(ORE, SInfo, StackTag, UARTag, DT, PDT, LI);
1543 }
1544
1545 // If we split the entry block, move any allocas that were originally in the
1546 // entry block back into the entry block so that they aren't treated as
1547 // dynamic allocas.
1548 if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1549 InsertPt = F.getEntryBlock().begin();
1550 for (Instruction &I :
1551 llvm::make_early_inc_range(Range&: *EntryIRB.GetInsertBlock())) {
1552 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
1553 if (isa<ConstantInt>(Val: AI->getArraySize()))
1554 I.moveBefore(BB&: F.getEntryBlock(), I: InsertPt);
1555 }
1556 }
1557
1558 DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
1559 PostDominatorTree *PDT = FAM.getCachedResult<PostDominatorTreeAnalysis>(IR&: F);
1560 LoopInfo *LI = FAM.getCachedResult<LoopAnalysis>(IR&: F);
1561 DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
1562 const DataLayout &DL = F.getDataLayout();
1563 for (auto &Operand : OperandsToInstrument)
1564 instrumentMemAccess(O&: Operand, DTU, LI, DL);
1565 DTU.flush();
1566
1567 if (Opts.hwasan_instrument_mem_intrinsics && !IntrinToInstrument.empty()) {
1568 for (auto *Inst : IntrinToInstrument)
1569 instrumentMemIntrinsic(MI: Inst);
1570 }
1571
1572 ShadowBase = nullptr;
1573 StackBaseTag = nullptr;
1574 CachedFP = nullptr;
1575}
1576
1577void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1578 assert(!UsePageAliases);
1579 Constant *Initializer = GV->getInitializer();
1580 uint64_t SizeInBytes =
1581 M.getDataLayout().getTypeAllocSize(Ty: Initializer->getType());
1582 uint64_t NewSize = alignTo(Size: SizeInBytes, A: Mapping.getObjectAlignment());
1583 if (SizeInBytes != NewSize) {
1584 // Pad the initializer out to the next multiple of 16 bytes and add the
1585 // required short granule tag.
1586 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1587 Init.back() = Tag;
1588 Constant *Padding = ConstantDataArray::get(Context&: *C, Elts&: Init);
1589 Initializer = ConstantStruct::getAnon(V: {Initializer, Padding});
1590 }
1591
1592 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1593 GlobalValue::ExternalLinkage, Initializer,
1594 GV->getName() + ".hwasan");
1595 NewGV->copyAttributesFrom(Src: GV);
1596 NewGV->setLinkage(GlobalValue::PrivateLinkage);
1597 NewGV->copyMetadata(Src: GV, Offset: 0);
1598 NewGV->setAlignment(
1599 std::max(a: GV->getAlign().valueOrOne(), b: Mapping.getObjectAlignment()));
1600
1601 // It is invalid to ICF two globals that have different tags. In the case
1602 // where the size of the global is a multiple of the tag granularity the
1603 // contents of the globals may be the same but the tags (i.e. symbol values)
1604 // may be different, and the symbols are not considered during ICF. In the
1605 // case where the size is not a multiple of the granularity, the short granule
1606 // tags would discriminate two globals with different tags, but there would
1607 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1608 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1609 // granule tag in the last byte.
1610 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1611
1612 // Descriptor format (assuming little-endian):
1613 // bytes 0-3: relative address of global
1614 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1615 // it isn't, we create multiple descriptors)
1616 // byte 7: tag
1617 auto *DescriptorTy = StructType::get(elt1: Int32Ty, elts: Int32Ty);
1618 const uint64_t MaxDescriptorSize = 0xfffff0;
1619 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1620 DescriptorPos += MaxDescriptorSize) {
1621 auto *Descriptor =
1622 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1623 nullptr, GV->getName() + ".hwasan.descriptor");
1624 auto *GVRelPtr = ConstantExpr::getTrunc(
1625 C: ConstantExpr::getAdd(
1626 C1: ConstantExpr::getSub(
1627 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1628 C2: ConstantExpr::getPtrToInt(C: Descriptor, Ty: Int64Ty)),
1629 C2: ConstantInt::get(Ty: Int64Ty, V: DescriptorPos)),
1630 Ty: Int32Ty);
1631 uint32_t Size = std::min(a: SizeInBytes - DescriptorPos, b: MaxDescriptorSize);
1632 auto *SizeAndTag = ConstantInt::get(Ty: Int32Ty, V: Size | (uint32_t(Tag) << 24));
1633 Descriptor->setComdat(NewGV->getComdat());
1634 Descriptor->setInitializer(ConstantStruct::getAnon(V: {GVRelPtr, SizeAndTag}));
1635 Descriptor->setSection("hwasan_globals");
1636 Descriptor->setMetadata(KindID: LLVMContext::MD_associated,
1637 Node: MDNode::get(Context&: *C, MDs: ValueAsMetadata::get(V: NewGV)));
1638 appendToCompilerUsed(M, Values: Descriptor);
1639 }
1640
1641 Constant *Aliasee = ConstantExpr::getIntToPtr(
1642 C: ConstantExpr::getAdd(
1643 C1: ConstantExpr::getPtrToInt(C: NewGV, Ty: Int64Ty),
1644 C2: ConstantInt::get(Ty: Int64Ty, V: uint64_t(Tag) << PointerTagShift)),
1645 Ty: GV->getType());
1646 auto *Alias = GlobalAlias::create(Ty: GV->getValueType(), AddressSpace: GV->getAddressSpace(),
1647 Linkage: GV->getLinkage(), Name: "", Aliasee, Parent: &M);
1648 Alias->setVisibility(GV->getVisibility());
1649 Alias->takeName(V: GV);
1650 GV->replaceAllUsesWith(V: Alias);
1651 GV->eraseFromParent();
1652}
1653
1654void HWAddressSanitizer::instrumentGlobals() {
1655 std::vector<GlobalVariable *> Globals;
1656 for (GlobalVariable &GV : M.globals()) {
1657 if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1658 continue;
1659
1660 if (GV.isDeclarationForLinker() || GV.getName().starts_with(Prefix: "llvm.") ||
1661 GV.isThreadLocal())
1662 continue;
1663
1664 // Common symbols can't have aliases point to them, so they can't be tagged.
1665 if (GV.hasCommonLinkage())
1666 continue;
1667
1668 if (Opts.hwasan_all_globals) {
1669 // Avoid instrumenting intrinsic global variables.
1670 if (GV.getSection() == "llvm.metadata")
1671 continue;
1672 } else {
1673 // Globals with custom sections may be used in __start_/__stop_
1674 // enumeration, which would be broken both by adding tags and potentially
1675 // by the extra padding/alignment that we insert.
1676 if (GV.hasSection())
1677 continue;
1678 }
1679
1680 Globals.push_back(x: &GV);
1681 }
1682
1683 MD5 Hasher;
1684 Hasher.update(Str: M.getSourceFileName());
1685 MD5::MD5Result Hash;
1686 Hasher.final(Result&: Hash);
1687 uint8_t Tag = Hash[0];
1688
1689 assert(TagMaskByte >= 16);
1690
1691 for (GlobalVariable *GV : Globals) {
1692 // Don't allow globals to be tagged with something that looks like a
1693 // short-granule tag, otherwise we lose inter-granule overflow detection, as
1694 // the fast path shadow-vs-address check succeeds.
1695 if (Tag < 16 || Tag > TagMaskByte)
1696 Tag = 16;
1697 instrumentGlobal(GV, Tag: Tag++);
1698 }
1699}
1700
1701void HWAddressSanitizer::instrumentPersonalityFunctions() {
1702 // We need to untag stack frames as we unwind past them. That is the job of
1703 // the personality function wrapper, which either wraps an existing
1704 // personality function or acts as a personality function on its own. Each
1705 // function that has a personality function or that can be unwound past has
1706 // its personality function changed to a thunk that calls the personality
1707 // function wrapper in the runtime.
1708 MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1709 for (Function &F : M) {
1710 if (F.isDeclaration() || !F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
1711 continue;
1712
1713 if (F.hasPersonalityFn()) {
1714 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(x: &F);
1715 } else if (!F.hasFnAttribute(Kind: Attribute::NoUnwind)) {
1716 PersonalityFns[nullptr].push_back(x: &F);
1717 }
1718 }
1719
1720 if (PersonalityFns.empty())
1721 return;
1722
1723 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1724 Name: "__hwasan_personality_wrapper", RetTy: Int32Ty, Args: Int32Ty, Args: Int32Ty, Args: Int64Ty, Args: PtrTy,
1725 Args: PtrTy, Args: PtrTy, Args: PtrTy, Args: PtrTy);
1726 FunctionCallee UnwindGetGR = M.getOrInsertFunction(Name: "_Unwind_GetGR", RetTy: VoidTy);
1727 FunctionCallee UnwindGetCFA = M.getOrInsertFunction(Name: "_Unwind_GetCFA", RetTy: VoidTy);
1728
1729 for (auto &P : PersonalityFns) {
1730 std::string ThunkName = kHwasanPersonalityThunkName;
1731 if (P.first)
1732 ThunkName += ("." + P.first->getName()).str();
1733 FunctionType *ThunkFnTy = FunctionType::get(
1734 Result: Int32Ty, Params: {Int32Ty, Int32Ty, Int64Ty, PtrTy, PtrTy}, isVarArg: false);
1735 bool IsLocal = P.first && (!isa<GlobalValue>(Val: P.first) ||
1736 cast<GlobalValue>(Val: P.first)->hasLocalLinkage());
1737 auto *ThunkFn = Function::Create(Ty: ThunkFnTy,
1738 Linkage: IsLocal ? GlobalValue::InternalLinkage
1739 : GlobalValue::LinkOnceODRLinkage,
1740 N: ThunkName, M: &M);
1741 // TODO: think about other attributes as well.
1742 if (any_of(Range&: P.second, P: [](const Function *F) {
1743 return F->hasFnAttribute(Kind: "branch-target-enforcement");
1744 })) {
1745 ThunkFn->addFnAttr(Kind: "branch-target-enforcement");
1746 }
1747 if (!IsLocal) {
1748 ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1749 ThunkFn->setComdat(M.getOrInsertComdat(Name: ThunkName));
1750 }
1751
1752 auto *BB = BasicBlock::Create(Context&: *C, Name: "entry", Parent: ThunkFn);
1753 IRBuilder<> IRB(BB);
1754 CallInst *WrapperCall = IRB.CreateCall(
1755 Callee: HwasanPersonalityWrapper,
1756 Args: {ThunkFn->getArg(i: 0), ThunkFn->getArg(i: 1), ThunkFn->getArg(i: 2),
1757 ThunkFn->getArg(i: 3), ThunkFn->getArg(i: 4),
1758 P.first ? P.first : Constant::getNullValue(Ty: PtrTy),
1759 UnwindGetGR.getCallee(), UnwindGetCFA.getCallee()});
1760 WrapperCall->setTailCall();
1761 IRB.CreateRet(V: WrapperCall);
1762
1763 for (Function *F : P.second)
1764 F->setPersonalityFn(ThunkFn);
1765 }
1766}
1767
1768void HWAddressSanitizer::ShadowMapping::init(const InstrumentationOptions &Opts,
1769 Triple &TargetTriple,
1770 bool InstrumentWithCalls,
1771 bool CompileKernel) {
1772 // Start with defaults.
1773 Scale = kDefaultShadowScale;
1774 Kind = OffsetKind::kTls;
1775 WithFrameRecord = true;
1776
1777 // Tune for the target.
1778 if (TargetTriple.isOSFuchsia()) {
1779 // Fuchsia is always PIE, which means that the beginning of the address
1780 // space is always available.
1781 Kind = OffsetKind::kGlobal;
1782 } else if (CompileKernel || InstrumentWithCalls) {
1783 SetFixed(0);
1784 WithFrameRecord = false;
1785 }
1786
1787 WithFrameRecord = valueOr(X: Opts.hwasan_with_frame_record, Default: WithFrameRecord);
1788
1789 // Apply the last of ClMappingOffset and ClMappingOffsetDynamic.
1790 if (ClMappingOffsetDynamic.getNumOccurrences())
1791 Kind = ClMappingOffsetDynamic;
1792 if (ClMappingOffset.getNumOccurrences() > 0 &&
1793 !(ClMappingOffsetDynamic.getNumOccurrences() > 0 &&
1794 ClMappingOffsetDynamic.getPosition() > ClMappingOffset.getPosition())) {
1795 SetFixed(ClMappingOffset);
1796 }
1797}
1798