1//===- AddressSanitizer.cpp - memory error detector -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address basic correctness
10// checker.
11// Details of the algorithm:
12// https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
13//
14// FIXME: This sanitizer does not yet handle scalable vectors
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DepthFirstIterator.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/ADT/Twine.h"
28#include "llvm/Analysis/GlobalsModRef.h"
29#include "llvm/Analysis/MemoryBuiltins.h"
30#include "llvm/Analysis/StackSafetyAnalysis.h"
31#include "llvm/Analysis/TargetLibraryInfo.h"
32#include "llvm/Analysis/TargetTransformInfo.h"
33#include "llvm/Analysis/ValueTracking.h"
34#include "llvm/BinaryFormat/MachO.h"
35#include "llvm/Demangle/Demangle.h"
36#include "llvm/IR/Argument.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/BasicBlock.h"
39#include "llvm/IR/Comdat.h"
40#include "llvm/IR/Constant.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DIBuilder.h"
43#include "llvm/IR/DataLayout.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/EHPersonalities.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GlobalAlias.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/GlobalVariable.h"
52#include "llvm/IR/IRBuilder.h"
53#include "llvm/IR/InlineAsm.h"
54#include "llvm/IR/InstVisitor.h"
55#include "llvm/IR/InstrTypes.h"
56#include "llvm/IR/Instruction.h"
57#include "llvm/IR/Instructions.h"
58#include "llvm/IR/IntrinsicInst.h"
59#include "llvm/IR/Intrinsics.h"
60#include "llvm/IR/LLVMContext.h"
61#include "llvm/IR/MDBuilder.h"
62#include "llvm/IR/Metadata.h"
63#include "llvm/IR/Module.h"
64#include "llvm/IR/Type.h"
65#include "llvm/IR/Use.h"
66#include "llvm/IR/Value.h"
67#include "llvm/MC/MCSectionMachO.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Debug.h"
71#include "llvm/Support/ErrorHandling.h"
72#include "llvm/Support/MathExtras.h"
73#include "llvm/Support/ModRef.h"
74#include "llvm/Support/raw_ostream.h"
75#include "llvm/TargetParser/Triple.h"
76#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
77#include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
78#include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
79#include "llvm/Transforms/Utils/BasicBlockUtils.h"
80#include "llvm/Transforms/Utils/Instrumentation.h"
81#include "llvm/Transforms/Utils/Local.h"
82#include "llvm/Transforms/Utils/ModuleUtils.h"
83#include "llvm/Transforms/Utils/PromoteMemToReg.h"
84#include <algorithm>
85#include <cassert>
86#include <cstddef>
87#include <cstdint>
88#include <iomanip>
89#include <limits>
90#include <sstream>
91#include <string>
92#include <tuple>
93#include <utility>
94
95using namespace llvm;
96
97#define DEBUG_TYPE "asan"
98
99static const uint64_t kDefaultShadowScale = 3;
100static const uint64_t kDefaultShadowOffset32 = 1ULL << 29;
101static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
102static const uint64_t kDynamicShadowSentinel =
103 std::numeric_limits<uint64_t>::max();
104static const uint64_t kSmallX86_64ShadowOffsetBase = 0x7FFFFFFF; // < 2G.
105static const uint64_t kSmallX86_64ShadowOffsetAlignMask = ~0xFFFULL;
106static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000;
107static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 44;
108static const uint64_t kSystemZ_ShadowOffset64 = 1ULL << 52;
109static const uint64_t kMIPS_ShadowOffsetN32 = 1ULL << 29;
110static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
111static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
112static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
113static const uint64_t kLoongArch64_ShadowOffset64 = 1ULL << 46;
114static const uint64_t kRISCV64_ShadowOffset64 = kDynamicShadowSentinel;
115static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
116static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
117static const uint64_t kFreeBSDAArch64_ShadowOffset64 = 1ULL << 47;
118static const uint64_t kFreeBSDKasan_ShadowOffset64 = 0xdffff7c000000000;
119static const uint64_t kNetBSD_ShadowOffset32 = 1ULL << 30;
120static const uint64_t kNetBSD_ShadowOffset64 = 1ULL << 46;
121static const uint64_t kNetBSDKasan_ShadowOffset64 = 0xdfff900000000000;
122static const uint64_t kPS_ShadowOffset64 = 1ULL << 40;
123static const uint64_t kWindowsShadowOffset32 = 3ULL << 28;
124static const uint64_t kWebAssemblyShadowOffset = 0;
125
126// The shadow memory space is dynamically allocated.
127static const uint64_t kWindowsShadowOffset64 = kDynamicShadowSentinel;
128
129static const size_t kMinStackMallocSize = 1 << 6; // 64B
130static const size_t kMaxStackMallocSize = 1 << 16; // 64K
131static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3;
132static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
133
134const char kAsanModuleCtorName[] = "asan.module_ctor";
135const char kAsanModuleDtorName[] = "asan.module_dtor";
136static const uint64_t kAsanCtorAndDtorPriority = 1;
137// On Emscripten, the system needs more than one priorities for constructors.
138static const uint64_t kAsanEmscriptenCtorAndDtorPriority = 50;
139const char kAsanReportErrorTemplate[] = "__asan_report_";
140const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
141const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
142const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
143const char kAsanUnregisterImageGlobalsName[] =
144 "__asan_unregister_image_globals";
145const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
146const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
147const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
148const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
149const char kAsanInitName[] = "__asan_init";
150const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
151const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
152const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
153const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
154static const int kMaxAsanStackMallocSizeClass = 10;
155const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
156const char kAsanStackMallocAlwaysNameTemplate[] =
157 "__asan_stack_malloc_always_";
158const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
159const char kAsanGenPrefix[] = "___asan_gen_";
160const char kODRGenPrefix[] = "__odr_asan_gen_";
161const char kSanCovGenPrefix[] = "__sancov_gen_";
162const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
163const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
164const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
165
166// ASan version script has __asan_* wildcard. Triple underscore prevents a
167// linker (gold) warning about attempting to export a local symbol.
168const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
169
170const char kAsanOptionDetectUseAfterReturn[] =
171 "__asan_option_detect_stack_use_after_return";
172
173const char kAsanShadowMemoryDynamicAddress[] =
174 "__asan_shadow_memory_dynamic_address";
175
176const char kAsanAllocaPoison[] = "__asan_alloca_poison";
177const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
178
179const char kAMDGPUAddressSharedName[] = "llvm.amdgcn.is.shared";
180const char kAMDGPUAddressPrivateName[] = "llvm.amdgcn.is.private";
181const char kAMDGPUBallotName[] = "llvm.amdgcn.ballot.i64";
182const char kAMDGPUUnreachableName[] = "llvm.amdgcn.unreachable";
183
184// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
185static const size_t kNumberOfAccessSizes = 5;
186
187static const uint64_t kAllocaRzSize = 32;
188
189// ASanAccessInfo implementation constants.
190constexpr size_t kCompileKernelShift = 0;
191constexpr size_t kCompileKernelMask = 0x1;
192constexpr size_t kAccessSizeIndexShift = 1;
193constexpr size_t kAccessSizeIndexMask = 0xf;
194constexpr size_t kIsWriteShift = 5;
195constexpr size_t kIsWriteMask = 0x1;
196
197// Command-line flags.
198
199static cl::opt<bool> ClEnableKasan(
200 "asan-kernel", cl::desc("Enable KernelAddressSanitizer instrumentation"),
201 cl::Hidden, cl::init(Val: false));
202
203static cl::opt<bool> ClRecover(
204 "asan-recover",
205 cl::desc("Enable recovery mode (continue-after-error)."),
206 cl::Hidden, cl::init(Val: false));
207
208static cl::opt<bool> ClInsertVersionCheck(
209 "asan-guard-against-version-mismatch",
210 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
211 cl::init(Val: true));
212
213// This flag may need to be replaced with -f[no-]asan-reads.
214static cl::opt<bool> ClInstrumentReads("asan-instrument-reads",
215 cl::desc("instrument read instructions"),
216 cl::Hidden, cl::init(Val: true));
217
218static cl::opt<bool> ClInstrumentWrites(
219 "asan-instrument-writes", cl::desc("instrument write instructions"),
220 cl::Hidden, cl::init(Val: true));
221
222static cl::opt<bool>
223 ClUseStackSafety("asan-use-stack-safety", cl::Hidden, cl::init(Val: true),
224 cl::Hidden, cl::desc("Use Stack Safety analysis results"));
225
226static cl::opt<bool> ClInstrumentAtomics(
227 "asan-instrument-atomics",
228 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
229 cl::init(Val: true));
230
231static cl::opt<bool>
232 ClInstrumentByval("asan-instrument-byval",
233 cl::desc("instrument byval call arguments"), cl::Hidden,
234 cl::init(Val: true));
235
236static cl::opt<bool> ClAlwaysSlowPath(
237 "asan-always-slow-path",
238 cl::desc("use instrumentation with slow path for all accesses"), cl::Hidden,
239 cl::init(Val: false));
240
241static cl::opt<bool> ClForceDynamicShadow(
242 "asan-force-dynamic-shadow",
243 cl::desc("Load shadow address into a local variable for each function"),
244 cl::Hidden, cl::init(Val: false));
245
246static cl::opt<bool>
247 ClWithIfunc("asan-with-ifunc",
248 cl::desc("Access dynamic shadow through an ifunc global on "
249 "platforms that support this"),
250 cl::Hidden, cl::init(Val: true));
251
252static cl::opt<int>
253 ClShadowAddrSpace("asan-shadow-addr-space",
254 cl::desc("Address space for pointers to the shadow map"),
255 cl::Hidden, cl::init(Val: 0));
256
257static cl::opt<bool> ClWithIfuncSuppressRemat(
258 "asan-with-ifunc-suppress-remat",
259 cl::desc("Suppress rematerialization of dynamic shadow address by passing "
260 "it through inline asm in prologue."),
261 cl::Hidden, cl::init(Val: true));
262
263// This flag limits the number of instructions to be instrumented
264// in any given BB. Normally, this should be set to unlimited (INT_MAX),
265// but due to http://llvm.org/bugs/show_bug.cgi?id=12652 we temporary
266// set it to 10000.
267static cl::opt<int> ClMaxInsnsToInstrumentPerBB(
268 "asan-max-ins-per-bb", cl::init(Val: 10000),
269 cl::desc("maximal number of instructions to instrument in any given BB"),
270 cl::Hidden);
271
272// This flag may need to be replaced with -f[no]asan-stack.
273static cl::opt<bool> ClStack("asan-stack", cl::desc("Handle stack memory"),
274 cl::Hidden, cl::init(Val: true));
275static cl::opt<uint32_t> ClMaxInlinePoisoningSize(
276 "asan-max-inline-poisoning-size",
277 cl::desc(
278 "Inline shadow poisoning for blocks up to the given size in bytes."),
279 cl::Hidden, cl::init(Val: 64));
280
281static cl::opt<AsanDetectStackUseAfterReturnMode> ClUseAfterReturn(
282 "asan-use-after-return",
283 cl::desc("Sets the mode of detection for stack-use-after-return."),
284 cl::values(
285 clEnumValN(AsanDetectStackUseAfterReturnMode::Never, "never",
286 "Never detect stack use after return."),
287 clEnumValN(
288 AsanDetectStackUseAfterReturnMode::Runtime, "runtime",
289 "Detect stack use after return if "
290 "binary flag 'ASAN_OPTIONS=detect_stack_use_after_return' is set."),
291 clEnumValN(AsanDetectStackUseAfterReturnMode::Always, "always",
292 "Always detect stack use after return.")),
293 cl::Hidden, cl::init(Val: AsanDetectStackUseAfterReturnMode::Runtime));
294
295static cl::opt<bool> ClRedzoneByvalArgs("asan-redzone-byval-args",
296 cl::desc("Create redzones for byval "
297 "arguments (extra copy "
298 "required)"), cl::Hidden,
299 cl::init(Val: true));
300
301static cl::opt<bool> ClUseAfterScope("asan-use-after-scope",
302 cl::desc("Check stack-use-after-scope"),
303 cl::Hidden, cl::init(Val: false));
304
305// This flag may need to be replaced with -f[no]asan-globals.
306static cl::opt<bool> ClGlobals("asan-globals",
307 cl::desc("Handle global objects"), cl::Hidden,
308 cl::init(Val: true));
309
310static cl::opt<bool> ClInitializers("asan-initialization-order",
311 cl::desc("Handle C++ initializer order"),
312 cl::Hidden, cl::init(Val: true));
313
314static cl::opt<bool> ClInvalidPointerPairs(
315 "asan-detect-invalid-pointer-pair",
316 cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden,
317 cl::init(Val: false));
318
319static cl::opt<bool> ClInvalidPointerCmp(
320 "asan-detect-invalid-pointer-cmp",
321 cl::desc("Instrument <, <=, >, >= with pointer operands"), cl::Hidden,
322 cl::init(Val: false));
323
324static cl::opt<bool> ClInvalidPointerSub(
325 "asan-detect-invalid-pointer-sub",
326 cl::desc("Instrument - operations with pointer operands"), cl::Hidden,
327 cl::init(Val: false));
328
329static cl::opt<unsigned> ClRealignStack(
330 "asan-realign-stack",
331 cl::desc("Realign stack to the value of this flag (power of two)"),
332 cl::Hidden, cl::init(Val: 32));
333
334static cl::opt<int> ClInstrumentationWithCallsThreshold(
335 "asan-instrumentation-with-call-threshold",
336 cl::desc("If the function being instrumented contains more than "
337 "this number of memory accesses, use callbacks instead of "
338 "inline checks (-1 means never use callbacks)."),
339 cl::Hidden, cl::init(Val: 7000));
340
341static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
342 "asan-memory-access-callback-prefix",
343 cl::desc("Prefix for memory access callbacks"), cl::Hidden,
344 cl::init(Val: "__asan_"));
345
346static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
347 "asan-kernel-mem-intrinsic-prefix",
348 cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
349 cl::init(Val: false));
350
351static cl::opt<bool>
352 ClInstrumentDynamicAllocas("asan-instrument-dynamic-allocas",
353 cl::desc("instrument dynamic allocas"),
354 cl::Hidden, cl::init(Val: true));
355
356static cl::opt<bool> ClSkipPromotableAllocas(
357 "asan-skip-promotable-allocas",
358 cl::desc("Do not instrument promotable allocas"), cl::Hidden,
359 cl::init(Val: true));
360
361static cl::opt<AsanCtorKind> ClConstructorKind(
362 "asan-constructor-kind",
363 cl::desc("Sets the ASan constructor kind"),
364 cl::values(clEnumValN(AsanCtorKind::None, "none", "No constructors"),
365 clEnumValN(AsanCtorKind::Global, "global",
366 "Use global constructors")),
367 cl::init(Val: AsanCtorKind::Global), cl::Hidden);
368// These flags allow to change the shadow mapping.
369// The shadow mapping looks like
370// Shadow = (Mem >> scale) + offset
371
372static cl::opt<int> ClMappingScale("asan-mapping-scale",
373 cl::desc("scale of asan shadow mapping"),
374 cl::Hidden, cl::init(Val: 0));
375
376static cl::opt<uint64_t>
377 ClMappingOffset("asan-mapping-offset",
378 cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"),
379 cl::Hidden, cl::init(Val: 0));
380
381// Optimization flags. Not user visible, used mostly for testing
382// and benchmarking the tool.
383
384static cl::opt<bool> ClOpt("asan-opt", cl::desc("Optimize instrumentation"),
385 cl::Hidden, cl::init(Val: true));
386
387static cl::opt<bool> ClOptimizeCallbacks("asan-optimize-callbacks",
388 cl::desc("Optimize callbacks"),
389 cl::Hidden, cl::init(Val: false));
390
391static cl::opt<bool> ClOptSameTemp(
392 "asan-opt-same-temp", cl::desc("Instrument the same temp just once"),
393 cl::Hidden, cl::init(Val: true));
394
395static cl::opt<bool> ClOptGlobals("asan-opt-globals",
396 cl::desc("Don't instrument scalar globals"),
397 cl::Hidden, cl::init(Val: true));
398
399static cl::opt<bool> ClOptStack(
400 "asan-opt-stack", cl::desc("Don't instrument scalar stack variables"),
401 cl::Hidden, cl::init(Val: false));
402
403static cl::opt<bool> ClDynamicAllocaStack(
404 "asan-stack-dynamic-alloca",
405 cl::desc("Use dynamic alloca to represent stack variables"), cl::Hidden,
406 cl::init(Val: true));
407
408static cl::opt<uint32_t> ClForceExperiment(
409 "asan-force-experiment",
410 cl::desc("Force optimization experiment (for testing)"), cl::Hidden,
411 cl::init(Val: 0));
412
413static cl::opt<bool>
414 ClUsePrivateAlias("asan-use-private-alias",
415 cl::desc("Use private aliases for global variables"),
416 cl::Hidden, cl::init(Val: true));
417
418static cl::opt<bool>
419 ClUseOdrIndicator("asan-use-odr-indicator",
420 cl::desc("Use odr indicators to improve ODR reporting"),
421 cl::Hidden, cl::init(Val: true));
422
423static cl::opt<bool>
424 ClUseGlobalsGC("asan-globals-live-support",
425 cl::desc("Use linker features to support dead "
426 "code stripping of globals"),
427 cl::Hidden, cl::init(Val: true));
428
429// This is on by default even though there is a bug in gold:
430// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
431static cl::opt<bool>
432 ClWithComdat("asan-with-comdat",
433 cl::desc("Place ASan constructors in comdat sections"),
434 cl::Hidden, cl::init(Val: true));
435
436static cl::opt<AsanDtorKind> ClOverrideDestructorKind(
437 "asan-destructor-kind",
438 cl::desc("Sets the ASan destructor kind. The default is to use the value "
439 "provided to the pass constructor"),
440 cl::values(clEnumValN(AsanDtorKind::None, "none", "No destructors"),
441 clEnumValN(AsanDtorKind::Global, "global",
442 "Use global destructors")),
443 cl::init(Val: AsanDtorKind::Invalid), cl::Hidden);
444
445static cl::list<unsigned> ClAddrSpaces(
446 "asan-instrument-address-spaces",
447 cl::desc("Only instrument variables in the specified address spaces."),
448 cl::Hidden, cl::CommaSeparated);
449
450// Debug flags.
451
452static cl::opt<int> ClDebugStack("asan-debug-stack", cl::desc("debug stack"),
453 cl::Hidden, cl::init(Val: 0));
454
455static cl::opt<std::string> ClDebugFunc("asan-debug-func", cl::Hidden,
456 cl::desc("Debug func"));
457
458static cl::opt<int> ClDebugMin("asan-debug-min", cl::desc("Debug min inst"),
459 cl::Hidden, cl::init(Val: -1));
460
461static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug max inst"),
462 cl::Hidden, cl::init(Val: -1));
463
464STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
465STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
466STATISTIC(NumOptimizedAccessesToGlobalVar,
467 "Number of optimized accesses to global vars");
468STATISTIC(NumOptimizedAccessesToStackVar,
469 "Number of optimized accesses to stack vars");
470
471namespace {
472
473/// This struct defines the shadow mapping using the rule:
474/// shadow = (mem >> Scale) ADD-or-OR Offset.
475/// If InGlobal is true, then
476/// extern char __asan_shadow[];
477/// shadow = (mem >> Scale) + &__asan_shadow
478struct ShadowMapping {
479 int Scale;
480 uint64_t Offset;
481 bool OrShadowOffset;
482 bool InGlobal;
483};
484
485} // end anonymous namespace
486
487static ShadowMapping getShadowMapping(const Triple &TargetTriple, int LongSize,
488 bool IsKasan) {
489 bool IsAndroid = TargetTriple.isAndroid();
490 bool IsIOS = TargetTriple.isiOS() || TargetTriple.isWatchOS() ||
491 TargetTriple.isDriverKit();
492 bool IsMacOS = TargetTriple.isMacOSX();
493 bool IsFreeBSD = TargetTriple.isOSFreeBSD();
494 bool IsNetBSD = TargetTriple.isOSNetBSD();
495 bool IsPS = TargetTriple.isPS();
496 bool IsLinux = TargetTriple.isOSLinux();
497 bool IsPPC64 = TargetTriple.getArch() == Triple::ppc64 ||
498 TargetTriple.getArch() == Triple::ppc64le;
499 bool IsSystemZ = TargetTriple.getArch() == Triple::systemz;
500 bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
501 bool IsMIPSN32ABI = TargetTriple.isABIN32();
502 bool IsMIPS32 = TargetTriple.isMIPS32();
503 bool IsMIPS64 = TargetTriple.isMIPS64();
504 bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb();
505 bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 ||
506 TargetTriple.getArch() == Triple::aarch64_be;
507 bool IsLoongArch64 = TargetTriple.isLoongArch64();
508 bool IsRISCV64 = TargetTriple.getArch() == Triple::riscv64;
509 bool IsWindows = TargetTriple.isOSWindows();
510 bool IsFuchsia = TargetTriple.isOSFuchsia();
511 bool IsAMDGPU = TargetTriple.isAMDGPU();
512 bool IsHaiku = TargetTriple.isOSHaiku();
513 bool IsWasm = TargetTriple.isWasm();
514 bool IsBPF = TargetTriple.isBPF();
515
516 ShadowMapping Mapping;
517
518 Mapping.Scale = kDefaultShadowScale;
519 if (ClMappingScale.getNumOccurrences() > 0) {
520 Mapping.Scale = ClMappingScale;
521 }
522
523 if (LongSize == 32) {
524 if (IsAndroid)
525 Mapping.Offset = kDynamicShadowSentinel;
526 else if (IsMIPSN32ABI)
527 Mapping.Offset = kMIPS_ShadowOffsetN32;
528 else if (IsMIPS32)
529 Mapping.Offset = kMIPS32_ShadowOffset32;
530 else if (IsFreeBSD)
531 Mapping.Offset = kFreeBSD_ShadowOffset32;
532 else if (IsNetBSD)
533 Mapping.Offset = kNetBSD_ShadowOffset32;
534 else if (IsIOS)
535 Mapping.Offset = kDynamicShadowSentinel;
536 else if (IsWindows)
537 Mapping.Offset = kWindowsShadowOffset32;
538 else if (IsWasm)
539 Mapping.Offset = kWebAssemblyShadowOffset;
540 else
541 Mapping.Offset = kDefaultShadowOffset32;
542 } else { // LongSize == 64
543 // Fuchsia is always PIE, which means that the beginning of the address
544 // space is always available.
545 if (IsFuchsia) {
546 // kDynamicShadowSentinel tells instrumentation to use the dynamic shadow.
547 Mapping.Offset = kDynamicShadowSentinel;
548 } else if (IsPPC64)
549 Mapping.Offset = kPPC64_ShadowOffset64;
550 else if (IsSystemZ)
551 Mapping.Offset = kSystemZ_ShadowOffset64;
552 else if (IsFreeBSD && IsAArch64)
553 Mapping.Offset = kFreeBSDAArch64_ShadowOffset64;
554 else if (IsFreeBSD && !IsMIPS64) {
555 if (IsKasan)
556 Mapping.Offset = kFreeBSDKasan_ShadowOffset64;
557 else
558 Mapping.Offset = kFreeBSD_ShadowOffset64;
559 } else if (IsNetBSD) {
560 if (IsKasan)
561 Mapping.Offset = kNetBSDKasan_ShadowOffset64;
562 else
563 Mapping.Offset = kNetBSD_ShadowOffset64;
564 } else if (IsPS)
565 Mapping.Offset = kPS_ShadowOffset64;
566 else if (IsLinux && IsX86_64) {
567 if (IsKasan)
568 Mapping.Offset = kLinuxKasan_ShadowOffset64;
569 else
570 Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
571 (kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
572 } else if (IsWindows && (IsX86_64 || IsAArch64)) {
573 Mapping.Offset = kWindowsShadowOffset64;
574 } else if (IsMIPS64)
575 Mapping.Offset = kMIPS64_ShadowOffset64;
576 else if (IsIOS)
577 Mapping.Offset = kDynamicShadowSentinel;
578 else if (IsMacOS && IsAArch64)
579 Mapping.Offset = kDynamicShadowSentinel;
580 else if (IsAArch64)
581 Mapping.Offset = kAArch64_ShadowOffset64;
582 else if (IsLoongArch64)
583 Mapping.Offset = kLoongArch64_ShadowOffset64;
584 else if (IsRISCV64)
585 Mapping.Offset = kRISCV64_ShadowOffset64;
586 else if (IsAMDGPU)
587 Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
588 (kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
589 else if (IsHaiku && IsX86_64)
590 Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
591 (kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
592 else if (IsBPF)
593 Mapping.Offset = kDynamicShadowSentinel;
594 else if (IsWasm)
595 Mapping.Offset = kWebAssemblyShadowOffset;
596 else
597 Mapping.Offset = kDefaultShadowOffset64;
598 }
599
600 if (ClForceDynamicShadow) {
601 Mapping.Offset = kDynamicShadowSentinel;
602 }
603
604 if (ClMappingOffset.getNumOccurrences() > 0) {
605 Mapping.Offset = ClMappingOffset;
606 }
607
608 // OR-ing shadow offset if more efficient (at least on x86) if the offset
609 // is a power of two, but on ppc64 and loongarch64 we have to use add since
610 // the shadow offset is not necessarily 1/8-th of the address space. On
611 // SystemZ, we could OR the constant in a single instruction, but it's more
612 // efficient to load it once and use indexed addressing.
613 Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 && !IsSystemZ && !IsPS &&
614 !IsRISCV64 && !IsLoongArch64 &&
615 !(Mapping.Offset & (Mapping.Offset - 1)) &&
616 Mapping.Offset != kDynamicShadowSentinel;
617 Mapping.InGlobal = ClWithIfunc && IsAndroid && IsArmOrThumb;
618
619 return Mapping;
620}
621
622void llvm::getAddressSanitizerParams(const Triple &TargetTriple, int LongSize,
623 bool IsKasan, uint64_t *ShadowBase,
624 int *MappingScale, bool *OrShadowOffset) {
625 auto Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan);
626 *ShadowBase = Mapping.Offset;
627 *MappingScale = Mapping.Scale;
628 *OrShadowOffset = Mapping.OrShadowOffset;
629}
630
631void llvm::removeASanIncompatibleFnAttributes(Function &F, bool ReadsArgMem) {
632 // Adding sanitizer checks invalidates previously inferred memory attributes.
633 //
634 // This is not only true for sanitized functions, because AttrInfer can
635 // infer those attributes on libc functions, which is not true if those
636 // are instrumented (Android) or intercepted.
637 //
638 // We might want to model ASan shadow memory more opaquely to get rid of
639 // this problem altogether, by hiding the shadow memory write in an
640 // intrinsic, essentially like in the AArch64StackTagging pass. But that's
641 // for another day.
642
643 bool Changed = false;
644 // We add memory(readwrite) to functions that don't already have that set and
645 // can access any non-inaccessible memory. Sanitizer instrumentation can
646 // read/write shadow memory, which is IRMemLocation::Other. Sanitizer
647 // instrumentation can instrument any memory accesses to non-inaccessible
648 // memory.
649 if (!F.getMemoryEffects()
650 .getWithoutLoc(Loc: IRMemLocation::InaccessibleMem)
651 .doesNotAccessMemory() &&
652 !isModAndRefSet(MRI: F.getMemoryEffects().getModRef(Loc: IRMemLocation::Other))) {
653 F.setMemoryEffects(F.getMemoryEffects() |
654 MemoryEffects::otherMemOnly(MR: ModRefInfo::ModRef));
655 Changed = true;
656 }
657 // HWASan reads from argument memory even for previously write-only accesses.
658 if (ReadsArgMem) {
659 if (F.getMemoryEffects().getModRef(Loc: IRMemLocation::ArgMem) ==
660 ModRefInfo::Mod) {
661 F.setMemoryEffects(F.getMemoryEffects() |
662 MemoryEffects::argMemOnly(MR: ModRefInfo::Ref));
663 Changed = true;
664 }
665 for (Argument &A : F.args()) {
666 if (A.hasAttribute(Kind: Attribute::WriteOnly)) {
667 A.removeAttr(Kind: Attribute::WriteOnly);
668 Changed = true;
669 }
670 }
671 }
672 if (Changed) {
673 // nobuiltin makes sure later passes don't restore assumptions about
674 // the function.
675 F.addFnAttr(Kind: Attribute::NoBuiltin);
676 }
677}
678
679ASanAccessInfo::ASanAccessInfo(int32_t Packed)
680 : Packed(Packed),
681 AccessSizeIndex((Packed >> kAccessSizeIndexShift) & kAccessSizeIndexMask),
682 IsWrite((Packed >> kIsWriteShift) & kIsWriteMask),
683 CompileKernel((Packed >> kCompileKernelShift) & kCompileKernelMask) {}
684
685ASanAccessInfo::ASanAccessInfo(bool IsWrite, bool CompileKernel,
686 uint8_t AccessSizeIndex)
687 : Packed((IsWrite << kIsWriteShift) +
688 (CompileKernel << kCompileKernelShift) +
689 (AccessSizeIndex << kAccessSizeIndexShift)),
690 AccessSizeIndex(AccessSizeIndex), IsWrite(IsWrite),
691 CompileKernel(CompileKernel) {}
692
693static uint64_t getRedzoneSizeForScale(int MappingScale) {
694 // Redzone used for stack and globals is at least 32 bytes.
695 // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
696 return std::max(a: 32U, b: 1U << MappingScale);
697}
698
699static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
700 if (TargetTriple.isOSEmscripten())
701 return kAsanEmscriptenCtorAndDtorPriority;
702 else
703 return kAsanCtorAndDtorPriority;
704}
705
706static Twine genName(StringRef suffix) {
707 return Twine(kAsanGenPrefix) + suffix;
708}
709
710namespace {
711
712class AsanFunctionInserter {
713public:
714 AsanFunctionInserter(Module &M) : M(M) {}
715
716 template <typename... ArgTypes>
717 FunctionCallee insertFunction(StringRef Name, ArgTypes &&...Args) {
718 return M.getOrInsertFunction(Name, std::forward<ArgTypes>(Args)...);
719 }
720
721private:
722 Module &M;
723};
724
725} // end anonymous namespace
726
727namespace {
728/// Helper RAII class to post-process inserted asan runtime calls during a
729/// pass on a single Function. Upon end of scope, detects and applies the
730/// required funclet OpBundle.
731class RuntimeCallInserter {
732 Function *OwnerFn = nullptr;
733 bool TrackInsertedCalls = false;
734 SmallVector<CallInst *> InsertedCalls;
735
736public:
737 RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {
738 if (Fn.hasPersonalityFn()) {
739 auto Personality = classifyEHPersonality(Pers: Fn.getPersonalityFn());
740 if (isScopedEHPersonality(Pers: Personality))
741 TrackInsertedCalls = true;
742 }
743 }
744
745 ~RuntimeCallInserter() {
746 if (InsertedCalls.empty())
747 return;
748 assert(TrackInsertedCalls && "Calls were wrongly tracked");
749
750 DenseMap<BasicBlock *, ColorVector> BlockColors = colorEHFunclets(F&: *OwnerFn);
751 for (CallInst *CI : InsertedCalls) {
752 BasicBlock *BB = CI->getParent();
753 assert(BB && "Instruction doesn't belong to a BasicBlock");
754 assert(BB->getParent() == OwnerFn &&
755 "Instruction doesn't belong to the expected Function!");
756
757 ColorVector &Colors = BlockColors[BB];
758 // funclet opbundles are only valid in monochromatic BBs.
759 // Note that unreachable BBs are seen as colorless by colorEHFunclets()
760 // and will be DCE'ed later.
761 if (Colors.empty())
762 continue;
763 if (Colors.size() != 1) {
764 OwnerFn->getContext().emitError(
765 ErrorStr: "Instruction's BasicBlock is not monochromatic");
766 continue;
767 }
768
769 BasicBlock *Color = Colors.front();
770 BasicBlock::iterator EHPadIt = Color->getFirstNonPHIIt();
771
772 if (EHPadIt != Color->end() && EHPadIt->isEHPad()) {
773 // Replace CI with a clone with an added funclet OperandBundle
774 OperandBundleDef OB("funclet", &*EHPadIt);
775 auto *NewCall = CallBase::addOperandBundle(CB: CI, ID: LLVMContext::OB_funclet,
776 OB, InsertPt: CI->getIterator());
777 NewCall->copyMetadata(SrcInst: *CI);
778 CI->replaceAllUsesWith(V: NewCall);
779 CI->eraseFromParent();
780 }
781 }
782 }
783
784 CallInst *createRuntimeCall(IRBuilder<> &IRB, FunctionCallee Callee,
785 ArrayRef<Value *> Args = {},
786 const Twine &Name = "") {
787 assert(IRB.GetInsertBlock()->getParent() == OwnerFn);
788
789 CallInst *Inst = IRB.CreateCall(Callee, Args, Name, FPMathTag: nullptr);
790 if (TrackInsertedCalls)
791 InsertedCalls.push_back(Elt: Inst);
792 return Inst;
793 }
794};
795
796/// AddressSanitizer: instrument the code in module to find memory bugs.
797struct AddressSanitizer {
798 AddressSanitizer(Module &M, const StackSafetyGlobalInfo *SSGI,
799 int InstrumentationWithCallsThreshold,
800 uint32_t MaxInlinePoisoningSize, bool CompileKernel = false,
801 bool Recover = false, bool UseAfterScope = false,
802 AsanDetectStackUseAfterReturnMode UseAfterReturn =
803 AsanDetectStackUseAfterReturnMode::Runtime)
804 : M(M), Inserter(M),
805 CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
806 : CompileKernel),
807 Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
808 UseAfterScope(UseAfterScope || ClUseAfterScope),
809 UseAfterReturn(ClUseAfterReturn.getNumOccurrences() ? ClUseAfterReturn
810 : UseAfterReturn),
811 SSGI(SSGI),
812 InstrumentationWithCallsThreshold(
813 ClInstrumentationWithCallsThreshold.getNumOccurrences() > 0
814 ? ClInstrumentationWithCallsThreshold
815 : InstrumentationWithCallsThreshold),
816 MaxInlinePoisoningSize(ClMaxInlinePoisoningSize.getNumOccurrences() > 0
817 ? ClMaxInlinePoisoningSize
818 : MaxInlinePoisoningSize) {
819 C = &(M.getContext());
820 DL = &M.getDataLayout();
821 LongSize = M.getDataLayout().getPointerSizeInBits();
822 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
823 PtrTy = PointerType::getUnqual(C&: *C);
824 Int32Ty = Type::getInt32Ty(C&: *C);
825 TargetTriple = M.getTargetTriple();
826
827 Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan: this->CompileKernel);
828
829 assert(this->UseAfterReturn != AsanDetectStackUseAfterReturnMode::Invalid);
830 }
831
832 TypeSize getAllocaSizeInBytes(const AllocaInst &AI) const {
833 return *AI.getAllocationSize(DL: AI.getDataLayout());
834 }
835
836 /// Check if we want (and can) handle this alloca.
837 bool isInterestingAlloca(const AllocaInst &AI);
838
839 bool ignoreAccess(Instruction *Inst, Value *Ptr);
840 void getInterestingMemoryOperands(
841 Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
842 const TargetTransformInfo *TTI);
843
844 void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
845 InterestingMemoryOperand &O, bool UseCalls,
846 const DataLayout &DL, RuntimeCallInserter &RTCI);
847 bool instrumentPointerComparisonOrSubtraction(Instruction *I,
848 RuntimeCallInserter &RTCI);
849 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
850 Value *Addr, MaybeAlign Alignment,
851 uint32_t TypeStoreSize, bool IsWrite,
852 Value *SizeArgument, bool UseCalls, uint32_t Exp,
853 RuntimeCallInserter &RTCI);
854 Instruction *instrumentAMDGPUAddress(Instruction *OrigIns,
855 Instruction *InsertBefore, Value *Addr,
856 uint32_t TypeStoreSize, bool IsWrite,
857 Value *SizeArgument);
858 Instruction *genAMDGPUReportBlock(IRBuilder<> &IRB, Value *Cond,
859 bool Recover);
860 void instrumentUnusualSizeOrAlignment(Instruction *I,
861 Instruction *InsertBefore, Value *Addr,
862 TypeSize TypeStoreSize, bool IsWrite,
863 Value *SizeArgument, bool UseCalls,
864 uint32_t Exp,
865 RuntimeCallInserter &RTCI);
866 void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL,
867 Type *IntptrTy, Value *Mask, Value *EVL,
868 Value *Stride, Instruction *I, Value *Addr,
869 MaybeAlign Alignment, unsigned Granularity,
870 Type *OpType, bool IsWrite,
871 Value *SizeArgument, bool UseCalls,
872 uint32_t Exp, RuntimeCallInserter &RTCI);
873 Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
874 Value *ShadowValue, uint32_t TypeStoreSize);
875 Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr,
876 bool IsWrite, size_t AccessSizeIndex,
877 Value *SizeArgument, uint32_t Exp,
878 RuntimeCallInserter &RTCI);
879 void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI);
880 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
881 bool suppressInstrumentationSiteForDebug(int &Instrumented);
882 bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI,
883 const TargetTransformInfo *TTI);
884 bool maybeInsertAsanInitAtFunctionEntry(Function &F);
885 bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
886 void markEscapedLocalAllocas(Function &F);
887 void markCatchParametersAsUninteresting(Function &F);
888
889private:
890 friend struct FunctionStackPoisoner;
891
892 void initializeCallbacks(const TargetLibraryInfo *TLI);
893
894 bool LooksLikeCodeInBug11395(Instruction *I);
895 bool GlobalIsLinkerInitialized(GlobalVariable *G);
896 bool isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, Value *Addr,
897 TypeSize TypeStoreSize) const;
898
899 /// Helper to cleanup per-function state.
900 struct FunctionStateRAII {
901 AddressSanitizer *Pass;
902
903 FunctionStateRAII(AddressSanitizer *Pass) : Pass(Pass) {
904 assert(Pass->ProcessedAllocas.empty() &&
905 "last pass forgot to clear cache");
906 assert(!Pass->LocalDynamicShadow);
907 }
908
909 ~FunctionStateRAII() {
910 Pass->LocalDynamicShadow = nullptr;
911 Pass->ProcessedAllocas.clear();
912 }
913 };
914
915 Module &M;
916 AsanFunctionInserter Inserter;
917 LLVMContext *C;
918 const DataLayout *DL;
919 Triple TargetTriple;
920 int LongSize;
921 bool CompileKernel;
922 bool Recover;
923 bool UseAfterScope;
924 AsanDetectStackUseAfterReturnMode UseAfterReturn;
925 Type *IntptrTy;
926 Type *Int32Ty;
927 PointerType *PtrTy;
928 ShadowMapping Mapping;
929 FunctionCallee AsanHandleNoReturnFunc;
930 FunctionCallee AsanPtrCmpFunction, AsanPtrSubFunction;
931 Constant *AsanShadowGlobal;
932
933 // These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize).
934 FunctionCallee AsanErrorCallback[2][2][kNumberOfAccessSizes];
935 FunctionCallee AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes];
936
937 // These arrays is indexed by AccessIsWrite and Experiment.
938 FunctionCallee AsanErrorCallbackSized[2][2];
939 FunctionCallee AsanMemoryAccessCallbackSized[2][2];
940
941 FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
942 Value *LocalDynamicShadow = nullptr;
943 const StackSafetyGlobalInfo *SSGI;
944 DenseMap<const AllocaInst *, bool> ProcessedAllocas;
945
946 FunctionCallee AMDGPUAddressShared;
947 FunctionCallee AMDGPUAddressPrivate;
948 int InstrumentationWithCallsThreshold;
949 uint32_t MaxInlinePoisoningSize;
950};
951
952class ModuleAddressSanitizer {
953public:
954 ModuleAddressSanitizer(Module &M, bool InsertVersionCheck,
955 bool CompileKernel = false, bool Recover = false,
956 bool UseGlobalsGC = true, bool UseOdrIndicator = true,
957 AsanDtorKind DestructorKind = AsanDtorKind::Global,
958 AsanCtorKind ConstructorKind = AsanCtorKind::Global)
959 : M(M), Inserter(M),
960 CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
961 : CompileKernel),
962 InsertVersionCheck(ClInsertVersionCheck.getNumOccurrences() > 0
963 ? ClInsertVersionCheck
964 : InsertVersionCheck),
965 Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
966 UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC && !this->CompileKernel),
967 // Enable aliases as they should have no downside with ODR indicators.
968 UsePrivateAlias(ClUsePrivateAlias.getNumOccurrences() > 0
969 ? ClUsePrivateAlias
970 : UseOdrIndicator),
971 UseOdrIndicator(ClUseOdrIndicator.getNumOccurrences() > 0
972 ? ClUseOdrIndicator
973 : UseOdrIndicator),
974 // Not a typo: ClWithComdat is almost completely pointless without
975 // ClUseGlobalsGC (because then it only works on modules without
976 // globals, which are rare); it is a prerequisite for ClUseGlobalsGC;
977 // and both suffer from gold PR19002 for which UseGlobalsGC constructor
978 // argument is designed as workaround. Therefore, disable both
979 // ClWithComdat and ClUseGlobalsGC unless the frontend says it's ok to
980 // do globals-gc.
981 UseCtorComdat(UseGlobalsGC && ClWithComdat && !this->CompileKernel),
982 DestructorKind(DestructorKind),
983 ConstructorKind(ClConstructorKind.getNumOccurrences() > 0
984 ? ClConstructorKind
985 : ConstructorKind) {
986 C = &(M.getContext());
987 int LongSize = M.getDataLayout().getPointerSizeInBits();
988 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
989 PtrTy = PointerType::getUnqual(C&: *C);
990 TargetTriple = M.getTargetTriple();
991 Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan: this->CompileKernel);
992
993 if (ClOverrideDestructorKind != AsanDtorKind::Invalid)
994 this->DestructorKind = ClOverrideDestructorKind;
995 assert(this->DestructorKind != AsanDtorKind::Invalid);
996 }
997
998 bool instrumentModule();
999
1000private:
1001 void initializeCallbacks();
1002
1003 void instrumentGlobals(IRBuilder<> &IRB, bool *CtorComdat);
1004 void InstrumentGlobalsCOFF(IRBuilder<> &IRB,
1005 ArrayRef<GlobalVariable *> ExtendedGlobals,
1006 ArrayRef<Constant *> MetadataInitializers);
1007 void instrumentGlobalsELF(IRBuilder<> &IRB,
1008 ArrayRef<GlobalVariable *> ExtendedGlobals,
1009 ArrayRef<Constant *> MetadataInitializers,
1010 const std::string &UniqueModuleId);
1011 void InstrumentGlobalsMachO(IRBuilder<> &IRB,
1012 ArrayRef<GlobalVariable *> ExtendedGlobals,
1013 ArrayRef<Constant *> MetadataInitializers);
1014 void
1015 InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB,
1016 ArrayRef<GlobalVariable *> ExtendedGlobals,
1017 ArrayRef<Constant *> MetadataInitializers);
1018
1019 GlobalVariable *CreateMetadataGlobal(Constant *Initializer,
1020 StringRef OriginalName);
1021 void SetComdatForGlobalMetadata(GlobalVariable *G, GlobalVariable *Metadata,
1022 StringRef InternalSuffix);
1023 Instruction *CreateAsanModuleDtor();
1024
1025 const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
1026 bool shouldInstrumentGlobal(GlobalVariable *G) const;
1027 bool ShouldUseMachOGlobalsSection() const;
1028 StringRef getGlobalMetadataSection() const;
1029 void poisonOneInitializer(Function &GlobalInit);
1030 void createInitializerPoisonCalls();
1031 uint64_t getMinRedzoneSizeForGlobal() const {
1032 return getRedzoneSizeForScale(MappingScale: Mapping.Scale);
1033 }
1034 uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
1035 int GetAsanVersion() const;
1036 GlobalVariable *getOrCreateModuleName();
1037
1038 Module &M;
1039 AsanFunctionInserter Inserter;
1040 bool CompileKernel;
1041 bool InsertVersionCheck;
1042 bool Recover;
1043 bool UseGlobalsGC;
1044 bool UsePrivateAlias;
1045 bool UseOdrIndicator;
1046 bool UseCtorComdat;
1047 AsanDtorKind DestructorKind;
1048 AsanCtorKind ConstructorKind;
1049 Type *IntptrTy;
1050 PointerType *PtrTy;
1051 LLVMContext *C;
1052 Triple TargetTriple;
1053 ShadowMapping Mapping;
1054 FunctionCallee AsanPoisonGlobals;
1055 FunctionCallee AsanUnpoisonGlobals;
1056 FunctionCallee AsanRegisterGlobals;
1057 FunctionCallee AsanUnregisterGlobals;
1058 FunctionCallee AsanRegisterImageGlobals;
1059 FunctionCallee AsanUnregisterImageGlobals;
1060 FunctionCallee AsanRegisterElfGlobals;
1061 FunctionCallee AsanUnregisterElfGlobals;
1062
1063 Function *AsanCtorFunction = nullptr;
1064 Function *AsanDtorFunction = nullptr;
1065 GlobalVariable *ModuleName = nullptr;
1066};
1067
1068// Stack poisoning does not play well with exception handling.
1069// When an exception is thrown, we essentially bypass the code
1070// that unpoisones the stack. This is why the run-time library has
1071// to intercept __cxa_throw (as well as longjmp, etc) and unpoison the entire
1072// stack in the interceptor. This however does not work inside the
1073// actual function which catches the exception. Most likely because the
1074// compiler hoists the load of the shadow value somewhere too high.
1075// This causes asan to report a non-existing bug on 453.povray.
1076// It sounds like an LLVM bug.
1077struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
1078 Function &F;
1079 AddressSanitizer &ASan;
1080 RuntimeCallInserter &RTCI;
1081 DIBuilder DIB;
1082 LLVMContext *C;
1083 Type *IntptrTy;
1084 Type *IntptrPtrTy;
1085 ShadowMapping Mapping;
1086
1087 SmallVector<AllocaInst *, 16> AllocaVec;
1088 SmallVector<AllocaInst *, 16> StaticAllocasToMoveUp;
1089 SmallVector<Instruction *, 8> RetVec;
1090
1091 FunctionCallee AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
1092 AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
1093 FunctionCallee AsanSetShadowFunc[0x100] = {};
1094 FunctionCallee AsanPoisonStackMemoryFunc, AsanUnpoisonStackMemoryFunc;
1095 FunctionCallee AsanAllocaPoisonFunc, AsanAllocasUnpoisonFunc;
1096
1097 // Stores a place and arguments of poisoning/unpoisoning call for alloca.
1098 struct AllocaPoisonCall {
1099 IntrinsicInst *InsBefore;
1100 AllocaInst *AI;
1101 uint64_t Size;
1102 bool DoPoison;
1103 };
1104 SmallVector<AllocaPoisonCall, 8> DynamicAllocaPoisonCallVec;
1105 SmallVector<AllocaPoisonCall, 8> StaticAllocaPoisonCallVec;
1106
1107 SmallVector<AllocaInst *, 1> DynamicAllocaVec;
1108 SmallVector<IntrinsicInst *, 1> StackRestoreVec;
1109 AllocaInst *DynamicAllocaLayout = nullptr;
1110 IntrinsicInst *LocalEscapeCall = nullptr;
1111
1112 bool HasInlineAsm = false;
1113 bool HasReturnsTwiceCall = false;
1114 bool PoisonStack;
1115
1116 FunctionStackPoisoner(Function &F, AddressSanitizer &ASan,
1117 RuntimeCallInserter &RTCI)
1118 : F(F), ASan(ASan), RTCI(RTCI),
1119 DIB(*F.getParent(), /*AllowUnresolved*/ false), C(ASan.C),
1120 IntptrTy(ASan.IntptrTy),
1121 IntptrPtrTy(PointerType::get(C&: IntptrTy->getContext(), AddressSpace: 0)),
1122 Mapping(ASan.Mapping),
1123 PoisonStack(ClStack && !F.getParent()->getTargetTriple().isAMDGPU()) {}
1124
1125 bool runOnFunction() {
1126 if (!PoisonStack)
1127 return false;
1128
1129 if (ClRedzoneByvalArgs)
1130 copyArgsPassedByValToAllocas();
1131
1132 // Collect alloca, ret, lifetime instructions etc.
1133 for (BasicBlock *BB : depth_first(G: &F.getEntryBlock())) visit(BB&: *BB);
1134
1135 if (AllocaVec.empty() && DynamicAllocaVec.empty()) return false;
1136
1137 initializeCallbacks(M&: *F.getParent());
1138
1139 processDynamicAllocas();
1140 processStaticAllocas();
1141
1142 if (ClDebugStack) {
1143 LLVM_DEBUG(dbgs() << F);
1144 }
1145 return true;
1146 }
1147
1148 // Arguments marked with the "byval" attribute are implicitly copied without
1149 // using an alloca instruction. To produce redzones for those arguments, we
1150 // copy them a second time into memory allocated with an alloca instruction.
1151 void copyArgsPassedByValToAllocas();
1152
1153 // Finds all Alloca instructions and puts
1154 // poisoned red zones around all of them.
1155 // Then unpoison everything back before the function returns.
1156 void processStaticAllocas();
1157 void processDynamicAllocas();
1158
1159 void createDynamicAllocasInitStorage();
1160
1161 // ----------------------- Visitors.
1162 /// Collect all Ret instructions, or the musttail call instruction if it
1163 /// precedes the return instruction.
1164 void visitReturnInst(ReturnInst &RI) {
1165 if (CallInst *CI = RI.getParent()->getTerminatingMustTailCall())
1166 RetVec.push_back(Elt: CI);
1167 else
1168 RetVec.push_back(Elt: &RI);
1169 }
1170
1171 /// Collect all Resume instructions.
1172 void visitResumeInst(ResumeInst &RI) { RetVec.push_back(Elt: &RI); }
1173
1174 /// Collect all CatchReturnInst instructions.
1175 void visitCleanupReturnInst(CleanupReturnInst &CRI) { RetVec.push_back(Elt: &CRI); }
1176
1177 void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore,
1178 Value *SavedStack) {
1179 IRBuilder<> IRB(InstBefore);
1180 Value *DynamicAreaPtr = IRB.CreatePtrToInt(V: SavedStack, DestTy: IntptrTy);
1181 // When we insert _asan_allocas_unpoison before @llvm.stackrestore, we
1182 // need to adjust extracted SP to compute the address of the most recent
1183 // alloca. We have a special @llvm.get.dynamic.area.offset intrinsic for
1184 // this purpose.
1185 if (!isa<ReturnInst>(Val: InstBefore)) {
1186 Value *DynamicAreaOffset = IRB.CreateIntrinsic(
1187 ID: Intrinsic::get_dynamic_area_offset, OverloadTypes: {IntptrTy}, Args: {});
1188
1189 DynamicAreaPtr = IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: SavedStack, DestTy: IntptrTy),
1190 RHS: DynamicAreaOffset);
1191 }
1192
1193 RTCI.createRuntimeCall(
1194 IRB, Callee: AsanAllocasUnpoisonFunc,
1195 Args: {IRB.CreateLoad(Ty: IntptrTy, Ptr: DynamicAllocaLayout), DynamicAreaPtr});
1196 }
1197
1198 // Unpoison dynamic allocas redzones.
1199 void unpoisonDynamicAllocas() {
1200 for (Instruction *Ret : RetVec)
1201 unpoisonDynamicAllocasBeforeInst(InstBefore: Ret, SavedStack: DynamicAllocaLayout);
1202
1203 for (Instruction *StackRestoreInst : StackRestoreVec)
1204 unpoisonDynamicAllocasBeforeInst(InstBefore: StackRestoreInst,
1205 SavedStack: StackRestoreInst->getOperand(i: 0));
1206 }
1207
1208 // Deploy and poison redzones around dynamic alloca call. To do this, we
1209 // should replace this call with another one with changed parameters and
1210 // replace all its uses with new address, so
1211 // addr = alloca type, old_size, align
1212 // is replaced by
1213 // new_size = (old_size + additional_size) * sizeof(type)
1214 // tmp = alloca i8, new_size, max(align, 32)
1215 // addr = tmp + 32 (first 32 bytes are for the left redzone).
1216 // Additional_size is added to make new memory allocation contain not only
1217 // requested memory, but also left, partial and right redzones.
1218 void handleDynamicAllocaCall(AllocaInst *AI);
1219
1220 /// Collect Alloca instructions we want (and can) handle.
1221 void visitAllocaInst(AllocaInst &AI) {
1222 // FIXME: Handle scalable vectors instead of ignoring them.
1223 if (!ASan.isInterestingAlloca(AI) || AI.isScalable()) {
1224 if (AI.isStaticAlloca()) {
1225 // Skip over allocas that are present *before* the first instrumented
1226 // alloca, we don't want to move those around.
1227 if (AllocaVec.empty())
1228 return;
1229
1230 StaticAllocasToMoveUp.push_back(Elt: &AI);
1231 }
1232 return;
1233 }
1234
1235 if (!AI.isStaticAlloca())
1236 DynamicAllocaVec.push_back(Elt: &AI);
1237 else
1238 AllocaVec.push_back(Elt: &AI);
1239 }
1240
1241 /// Collect lifetime intrinsic calls to check for use-after-scope
1242 /// errors.
1243 void visitIntrinsicInst(IntrinsicInst &II) {
1244 Intrinsic::ID ID = II.getIntrinsicID();
1245 if (ID == Intrinsic::stackrestore) StackRestoreVec.push_back(Elt: &II);
1246 if (ID == Intrinsic::localescape) LocalEscapeCall = &II;
1247 if (!ASan.UseAfterScope)
1248 return;
1249 if (!II.isLifetimeStartOrEnd())
1250 return;
1251 // Find alloca instruction that corresponds to llvm.lifetime argument.
1252 AllocaInst *AI = dyn_cast<AllocaInst>(Val: II.getArgOperand(i: 0));
1253 // We're interested only in allocas we can handle.
1254 if (!AI || !ASan.isInterestingAlloca(AI: *AI))
1255 return;
1256
1257 std::optional<TypeSize> Size = AI->getAllocationSize(DL: AI->getDataLayout());
1258 // Check that size is known and can be stored in IntptrTy.
1259 // TODO: Add support for scalable vectors if possible.
1260 if (!Size || Size->isScalable() ||
1261 !ConstantInt::isValueValidForType(Ty: IntptrTy, V: *Size))
1262 return;
1263
1264 bool DoPoison = (ID == Intrinsic::lifetime_end);
1265 AllocaPoisonCall APC = {.InsBefore: &II, .AI: AI, .Size: *Size, .DoPoison: DoPoison};
1266 if (AI->isStaticAlloca())
1267 StaticAllocaPoisonCallVec.push_back(Elt: APC);
1268 else if (ClInstrumentDynamicAllocas)
1269 DynamicAllocaPoisonCallVec.push_back(Elt: APC);
1270 }
1271
1272 void visitCallBase(CallBase &CB) {
1273 if (CallInst *CI = dyn_cast<CallInst>(Val: &CB)) {
1274 HasInlineAsm |= CI->isInlineAsm() && &CB != ASan.LocalDynamicShadow;
1275 HasReturnsTwiceCall |= CI->canReturnTwice();
1276 }
1277 }
1278
1279 // ---------------------- Helpers.
1280 void initializeCallbacks(Module &M);
1281
1282 // Copies bytes from ShadowBytes into shadow memory for indexes where
1283 // ShadowMask is not zero. If ShadowMask[i] is zero, we assume that
1284 // ShadowBytes[i] is constantly zero and doesn't need to be overwritten.
1285 void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
1286 IRBuilder<> &IRB, Value *ShadowBase);
1287 void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
1288 size_t Begin, size_t End, IRBuilder<> &IRB,
1289 Value *ShadowBase);
1290 void copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
1291 ArrayRef<uint8_t> ShadowBytes, size_t Begin,
1292 size_t End, IRBuilder<> &IRB, Value *ShadowBase);
1293
1294 void poisonAlloca(Value *V, uint64_t Size, IRBuilder<> &IRB, bool DoPoison);
1295
1296 Value *createAllocaForLayout(IRBuilder<> &IRB, const ASanStackFrameLayout &L,
1297 bool Dynamic);
1298 PHINode *createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue,
1299 Instruction *ThenTerm, Value *ValueIfFalse);
1300};
1301
1302} // end anonymous namespace
1303
1304void AddressSanitizerPass::printPipeline(
1305 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1306 static_cast<PassInfoMixin<AddressSanitizerPass> *>(this)->printPipeline(
1307 OS, MapClassName2PassName);
1308 OS << '<';
1309 if (Options.CompileKernel)
1310 OS << "kernel;";
1311 if (Options.UseAfterScope)
1312 OS << "use-after-scope";
1313 OS << '>';
1314}
1315
1316AddressSanitizerPass::AddressSanitizerPass(
1317 const AddressSanitizerOptions &Options, bool UseGlobalGC,
1318 bool UseOdrIndicator, AsanDtorKind DestructorKind,
1319 AsanCtorKind ConstructorKind)
1320 : Options(Options), UseGlobalGC(UseGlobalGC),
1321 UseOdrIndicator(UseOdrIndicator), DestructorKind(DestructorKind),
1322 ConstructorKind(ConstructorKind) {}
1323
1324PreservedAnalyses AddressSanitizerPass::run(Module &M,
1325 ModuleAnalysisManager &MAM) {
1326 // Return early if nosanitize_address module flag is present for the module.
1327 // This implies that asan pass has already run before.
1328 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_address"))
1329 return PreservedAnalyses::all();
1330
1331 ModuleAddressSanitizer ModuleSanitizer(
1332 M, Options.InsertVersionCheck, Options.CompileKernel, Options.Recover,
1333 UseGlobalGC, UseOdrIndicator, DestructorKind, ConstructorKind);
1334 bool Modified = false;
1335 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1336 const StackSafetyGlobalInfo *const SSGI =
1337 ClUseStackSafety ? &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M) : nullptr;
1338 for (Function &F : M) {
1339 if (F.empty())
1340 continue;
1341 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
1342 continue;
1343 if (!ClDebugFunc.empty() && ClDebugFunc == F.getName())
1344 continue;
1345 if (F.getName().starts_with(Prefix: "__asan_"))
1346 continue;
1347 if (F.isPresplitCoroutine())
1348 continue;
1349 AddressSanitizer FunctionSanitizer(
1350 M, SSGI, Options.InstrumentationWithCallsThreshold,
1351 Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
1352 Options.UseAfterScope, Options.UseAfterReturn);
1353 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1354 const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
1355 Modified |= FunctionSanitizer.instrumentFunction(F, TLI: &TLI, TTI: &TTI);
1356 }
1357 Modified |= ModuleSanitizer.instrumentModule();
1358 if (!Modified)
1359 return PreservedAnalyses::all();
1360
1361 PreservedAnalyses PA = PreservedAnalyses::none();
1362 // GlobalsAA is considered stateless and does not get invalidated unless
1363 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
1364 // make changes that require GlobalsAA to be invalidated.
1365 PA.abandon<GlobalsAA>();
1366 return PA;
1367}
1368
1369static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
1370 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
1371 assert(Res < kNumberOfAccessSizes);
1372 return Res;
1373}
1374
1375/// Check if \p G has been created by a trusted compiler pass.
1376static bool GlobalWasGeneratedByCompiler(GlobalVariable *G) {
1377 // Do not instrument @llvm.global_ctors, @llvm.used, etc.
1378 if (G->getName().starts_with(Prefix: "llvm.") ||
1379 // Do not instrument gcov counter arrays.
1380 G->getName().starts_with(Prefix: "__llvm_gcov_ctr") ||
1381 // Do not instrument rtti proxy symbols for function sanitizer.
1382 G->getName().starts_with(Prefix: "__llvm_rtti_proxy"))
1383 return true;
1384
1385 // Do not instrument asan globals.
1386 if (G->getName().starts_with(Prefix: kAsanGenPrefix) ||
1387 G->getName().starts_with(Prefix: kSanCovGenPrefix) ||
1388 G->getName().starts_with(Prefix: kODRGenPrefix))
1389 return true;
1390
1391 return false;
1392}
1393
1394static bool isUnsupportedAMDGPUAddrspace(Value *Addr) {
1395 Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
1396 unsigned int AddrSpace = PtrTy->getPointerAddressSpace();
1397 // Globals in address space 1 and 4 are supported for AMDGPU.
1398 if (AddrSpace == 3 || AddrSpace == 5)
1399 return true;
1400 return false;
1401}
1402
1403static bool isSupportedAddrspace(const Triple &TargetTriple, Value *Addr) {
1404 Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
1405 unsigned int AddrSpace = PtrTy->getPointerAddressSpace();
1406
1407 if (!ClAddrSpaces.empty())
1408 return is_contained(Range&: ClAddrSpaces, Element: AddrSpace);
1409
1410 if (TargetTriple.isAMDGPU())
1411 return !isUnsupportedAMDGPUAddrspace(Addr);
1412
1413 return AddrSpace == 0;
1414}
1415
1416Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
1417 if (TargetTriple.isOSDarwin() &&
1418 TargetTriple.getArch() == llvm::Triple::aarch64) {
1419 // Strip MTE-tag bits before translating to shadow address
1420 Shadow = IRB.CreateAnd(LHS: Shadow,
1421 RHS: ConstantInt::get(Ty: IntptrTy, V: ~(uint64_t(0x0f) << 56)));
1422 }
1423 // Shadow >> scale
1424 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
1425 if (Mapping.Offset == 0) return Shadow;
1426 // (Shadow >> scale) | offset
1427 Value *ShadowBase;
1428 if (LocalDynamicShadow)
1429 ShadowBase = LocalDynamicShadow;
1430 else
1431 ShadowBase = ConstantInt::get(Ty: IntptrTy, V: Mapping.Offset);
1432 if (Mapping.OrShadowOffset)
1433 return IRB.CreateOr(LHS: Shadow, RHS: ShadowBase);
1434 else
1435 return IRB.CreateAdd(LHS: Shadow, RHS: ShadowBase);
1436}
1437
1438// Instrument memset/memmove/memcpy
1439void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI,
1440 RuntimeCallInserter &RTCI) {
1441 InstrumentationIRBuilder IRB(MI);
1442 if (isa<MemTransferInst>(Val: MI)) {
1443 RTCI.createRuntimeCall(
1444 IRB, Callee: isa<MemMoveInst>(Val: MI) ? AsanMemmove : AsanMemcpy,
1445 Args: {IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 0), DestTy: PtrTy),
1446 IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 1), DestTy: PtrTy),
1447 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
1448 } else if (isa<MemSetInst>(Val: MI)) {
1449 RTCI.createRuntimeCall(
1450 IRB, Callee: AsanMemset,
1451 Args: {IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 0), DestTy: PtrTy),
1452 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1453 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
1454 }
1455 MI->eraseFromParent();
1456}
1457
1458/// Check if we want (and can) handle this alloca.
1459bool AddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1460 auto [It, Inserted] = ProcessedAllocas.try_emplace(Key: &AI);
1461
1462 if (!Inserted)
1463 return It->getSecond();
1464
1465 bool IsInteresting = // alloca() may be called with 0 size, ignore it.
1466 (((!AI.isStaticAlloca()) || !getAllocaSizeInBytes(AI).isZero()) &&
1467 // We are only interested in allocas not promotable to registers.
1468 // Promotable allocas are common under -O0.
1469 (!ClSkipPromotableAllocas || !isAllocaPromotable(AI: &AI)) &&
1470 // inalloca allocas are not treated as static, and we don't want
1471 // dynamic alloca instrumentation for them as well.
1472 !AI.isUsedWithInAlloca() &&
1473 // swifterror allocas are register promoted by ISel
1474 !AI.isSwiftError() &&
1475 // safe allocas are not interesting
1476 !(SSGI && SSGI->isSafe(AI)));
1477
1478 It->second = IsInteresting;
1479 return IsInteresting;
1480}
1481
1482bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
1483 // Check whether the target supports sanitizing the address space
1484 // of the pointer.
1485 if (!isSupportedAddrspace(TargetTriple, Addr: Ptr))
1486 return true;
1487
1488 // Ignore swifterror addresses.
1489 // swifterror memory addresses are mem2reg promoted by instruction
1490 // selection. As such they cannot have regular uses like an instrumentation
1491 // function and it makes no sense to track them as memory.
1492 if (Ptr->isSwiftError())
1493 return true;
1494
1495 // Treat memory accesses to promotable allocas as non-interesting since they
1496 // will not cause memory violations. This greatly speeds up the instrumented
1497 // executable at -O0.
1498 if (auto AI = dyn_cast_or_null<AllocaInst>(Val: Ptr))
1499 if (ClSkipPromotableAllocas && !isInterestingAlloca(AI: *AI))
1500 return true;
1501
1502 if (SSGI != nullptr && SSGI->stackAccessIsSafe(I: *Inst) &&
1503 findAllocaForValue(V: Ptr))
1504 return true;
1505
1506 return false;
1507}
1508
1509void AddressSanitizer::getInterestingMemoryOperands(
1510 Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
1511 const TargetTransformInfo *TTI) {
1512 // Do not instrument the load fetching the dynamic shadow address.
1513 if (LocalDynamicShadow == I)
1514 return;
1515
1516 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1517 if (!ClInstrumentReads || ignoreAccess(Inst: I, Ptr: LI->getPointerOperand()))
1518 return;
1519 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
1520 Args: LI->getType(), Args: LI->getAlign());
1521 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1522 if (!ClInstrumentWrites || ignoreAccess(Inst: I, Ptr: SI->getPointerOperand()))
1523 return;
1524 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
1525 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
1526 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
1527 if (!ClInstrumentAtomics || ignoreAccess(Inst: I, Ptr: RMW->getPointerOperand()))
1528 return;
1529 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
1530 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
1531 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1532 if (!ClInstrumentAtomics || ignoreAccess(Inst: I, Ptr: XCHG->getPointerOperand()))
1533 return;
1534 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
1535 Args: XCHG->getCompareOperand()->getType(),
1536 Args: std::nullopt);
1537 } else if (auto CI = dyn_cast<CallInst>(Val: I)) {
1538 switch (CI->getIntrinsicID()) {
1539 case Intrinsic::masked_load:
1540 case Intrinsic::masked_store:
1541 case Intrinsic::masked_gather:
1542 case Intrinsic::masked_scatter: {
1543 bool IsWrite = CI->getType()->isVoidTy();
1544 // Masked store has an initial operand for the value.
1545 unsigned OpOffset = IsWrite ? 1 : 0;
1546 if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1547 return;
1548
1549 auto BasePtr = CI->getOperand(i_nocapture: OpOffset);
1550 if (ignoreAccess(Inst: I, Ptr: BasePtr))
1551 return;
1552 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1553 MaybeAlign Alignment = CI->getParamAlign(ArgNo: 0);
1554 Value *Mask = CI->getOperand(i_nocapture: 1 + OpOffset);
1555 Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: Mask);
1556 break;
1557 }
1558 case Intrinsic::masked_expandload:
1559 case Intrinsic::masked_compressstore: {
1560 bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
1561 unsigned OpOffset = IsWrite ? 1 : 0;
1562 if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1563 return;
1564 auto BasePtr = CI->getOperand(i_nocapture: OpOffset);
1565 if (ignoreAccess(Inst: I, Ptr: BasePtr))
1566 return;
1567 MaybeAlign Alignment = BasePtr->getPointerAlignment(DL: *DL);
1568 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1569
1570 IRBuilder IB(I);
1571 Value *Mask = CI->getOperand(i_nocapture: 1 + OpOffset);
1572 // Use the popcount of Mask as the effective vector length.
1573 Type *ExtTy = VectorType::get(ElementType: IntptrTy, Other: cast<VectorType>(Val: Ty));
1574 Value *ExtMask = IB.CreateZExt(V: Mask, DestTy: ExtTy);
1575 Value *EVL = IB.CreateAddReduce(Src: ExtMask);
1576 Value *TrueMask = ConstantInt::get(Ty: Mask->getType(), V: 1);
1577 Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: TrueMask,
1578 Args&: EVL);
1579 break;
1580 }
1581 case Intrinsic::vp_load:
1582 case Intrinsic::vp_store:
1583 case Intrinsic::experimental_vp_strided_load:
1584 case Intrinsic::experimental_vp_strided_store: {
1585 auto *VPI = cast<VPIntrinsic>(Val: CI);
1586 unsigned IID = CI->getIntrinsicID();
1587 bool IsWrite = CI->getType()->isVoidTy();
1588 if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1589 return;
1590 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
1591 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1592 MaybeAlign Alignment = VPI->getOperand(i_nocapture: PtrOpNo)->getPointerAlignment(DL: *DL);
1593 Value *Stride = nullptr;
1594 if (IID == Intrinsic::experimental_vp_strided_store ||
1595 IID == Intrinsic::experimental_vp_strided_load) {
1596 Stride = VPI->getOperand(i_nocapture: PtrOpNo + 1);
1597 // Use the pointer alignment as the element alignment if the stride is a
1598 // multiple of the pointer alignment. Otherwise, the element alignment
1599 // should be Align(1).
1600 unsigned PointerAlign = Alignment.valueOrOne().value();
1601 if (!isa<ConstantInt>(Val: Stride) ||
1602 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
1603 Alignment = Align(1);
1604 }
1605 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
1606 Args: VPI->getMaskParam(), Args: VPI->getVectorLengthParam(),
1607 Args&: Stride);
1608 break;
1609 }
1610 case Intrinsic::vp_gather:
1611 case Intrinsic::vp_scatter: {
1612 auto *VPI = cast<VPIntrinsic>(Val: CI);
1613 unsigned IID = CI->getIntrinsicID();
1614 bool IsWrite = IID == Intrinsic::vp_scatter;
1615 if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1616 return;
1617 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
1618 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1619 MaybeAlign Alignment = VPI->getPointerAlignment();
1620 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
1621 Args: VPI->getMaskParam(),
1622 Args: VPI->getVectorLengthParam());
1623 break;
1624 }
1625 default:
1626 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1627 MemIntrinsicInfo IntrInfo;
1628 if (TTI->getTgtMemIntrinsic(Inst: II, Info&: IntrInfo))
1629 Interesting = IntrInfo.InterestingOperands;
1630 return;
1631 }
1632 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
1633 if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
1634 ignoreAccess(Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
1635 continue;
1636 Type *Ty = CI->getParamByValType(ArgNo);
1637 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
1638 }
1639 }
1640 }
1641}
1642
1643static bool isPointerOperand(Value *V) {
1644 return V->getType()->isPointerTy() || isa<PtrToIntInst, PtrToAddrInst>(Val: V);
1645}
1646
1647// This is a rough heuristic; it may cause both false positives and
1648// false negatives. The proper implementation requires cooperation with
1649// the frontend.
1650static bool isInterestingPointerComparison(Instruction *I) {
1651 if (ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: I)) {
1652 if (!Cmp->isRelational())
1653 return false;
1654 } else {
1655 return false;
1656 }
1657 return isPointerOperand(V: I->getOperand(i: 0)) &&
1658 isPointerOperand(V: I->getOperand(i: 1));
1659}
1660
1661// This is a rough heuristic; it may cause both false positives and
1662// false negatives. The proper implementation requires cooperation with
1663// the frontend.
1664static bool isInterestingPointerSubtraction(Instruction *I) {
1665 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: I)) {
1666 if (BO->getOpcode() != Instruction::Sub)
1667 return false;
1668 } else {
1669 return false;
1670 }
1671 return isPointerOperand(V: I->getOperand(i: 0)) &&
1672 isPointerOperand(V: I->getOperand(i: 1));
1673}
1674
1675bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
1676 // If a global variable does not have dynamic initialization we don't
1677 // have to instrument it. However, if a global does not have initializer
1678 // at all, we assume it has dynamic initializer (in other TU).
1679 if (!G->hasInitializer())
1680 return false;
1681
1682 if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().IsDynInit)
1683 return false;
1684
1685 return true;
1686}
1687
1688static bool isPointerPairOperand(Value *V, Type *IntptrTy) {
1689 Type *Ty = V->getType();
1690 if (Ty->isPtrOrPtrVectorTy())
1691 return true;
1692 return Ty->isIntOrIntVectorTy() &&
1693 Ty->getScalarSizeInBits() == IntptrTy->getScalarSizeInBits();
1694}
1695
1696bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
1697 Instruction *I, RuntimeCallInserter &RTCI) {
1698 Value *Param[2] = {I->getOperand(i: 0), I->getOperand(i: 1)};
1699 if (!isPointerPairOperand(V: Param[0], IntptrTy) ||
1700 !isPointerPairOperand(V: Param[1], IntptrTy))
1701 return false;
1702
1703 IRBuilder<> IRB(I);
1704 FunctionCallee F = isa<ICmpInst>(Val: I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
1705
1706 if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
1707 const auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
1708 // TODO: Add support for scalable vectors if possible.
1709 if (!VTy)
1710 return false;
1711
1712 assert(Param[0]->getType() == Param[1]->getType() &&
1713 "invalid vector pointer pair instrumentation operands");
1714 for (unsigned Index = 0, NumElements = VTy->getNumElements();
1715 Index != NumElements; ++Index) {
1716 Value *ScalarParam[2] = {
1717 IRB.CreatePointerCast(
1718 V: IRB.CreateExtractElement(Vec: Param[0], Idx: IRB.getInt32(C: Index)),
1719 DestTy: IntptrTy),
1720 IRB.CreatePointerCast(
1721 V: IRB.CreateExtractElement(Vec: Param[1], Idx: IRB.getInt32(C: Index)),
1722 DestTy: IntptrTy)};
1723 RTCI.createRuntimeCall(IRB, Callee: F, Args: ScalarParam);
1724 }
1725 return true;
1726 }
1727
1728 for (Value *&P : Param)
1729 P = IRB.CreatePointerCast(V: P, DestTy: IntptrTy);
1730 RTCI.createRuntimeCall(IRB, Callee: F, Args: Param);
1731 return true;
1732}
1733
1734static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
1735 Instruction *InsertBefore, Value *Addr,
1736 MaybeAlign Alignment, unsigned Granularity,
1737 TypeSize TypeStoreSize, bool IsWrite,
1738 Value *SizeArgument, bool UseCalls,
1739 uint32_t Exp, RuntimeCallInserter &RTCI) {
1740 // Instrument a 1-, 2-, 4-, 8-, or 16- byte access with one check
1741 // if the data is properly aligned.
1742 if (!TypeStoreSize.isScalable()) {
1743 const auto FixedSize = TypeStoreSize.getFixedValue();
1744 switch (FixedSize) {
1745 case 8:
1746 case 16:
1747 case 32:
1748 case 64:
1749 case 128:
1750 if (!Alignment || *Alignment >= Granularity ||
1751 *Alignment >= FixedSize / 8)
1752 return Pass->instrumentAddress(OrigIns: I, InsertBefore, Addr, Alignment,
1753 TypeStoreSize: FixedSize, IsWrite, SizeArgument: nullptr, UseCalls,
1754 Exp, RTCI);
1755 }
1756 }
1757 Pass->instrumentUnusualSizeOrAlignment(I, InsertBefore, Addr, TypeStoreSize,
1758 IsWrite, SizeArgument: nullptr, UseCalls, Exp, RTCI);
1759}
1760
1761void AddressSanitizer::instrumentMaskedLoadOrStore(
1762 AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask,
1763 Value *EVL, Value *Stride, Instruction *I, Value *Addr,
1764 MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite,
1765 Value *SizeArgument, bool UseCalls, uint32_t Exp,
1766 RuntimeCallInserter &RTCI) {
1767 auto *VTy = cast<VectorType>(Val: OpType);
1768 TypeSize ElemTypeSize = DL.getTypeStoreSizeInBits(Ty: VTy->getScalarType());
1769 auto Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
1770
1771 IRBuilder IB(I);
1772 Instruction *LoopInsertBefore = I;
1773 if (EVL) {
1774 // The end argument of SplitBlockAndInsertForLane is assumed bigger
1775 // than zero, so we should check whether EVL is zero here.
1776 Type *EVLType = EVL->getType();
1777 Value *IsEVLZero = IB.CreateICmpNE(LHS: EVL, RHS: ConstantInt::get(Ty: EVLType, V: 0));
1778 LoopInsertBefore = SplitBlockAndInsertIfThen(Cond: IsEVLZero, SplitBefore: I, Unreachable: false);
1779 IB.SetInsertPoint(LoopInsertBefore);
1780 // Cast EVL to IntptrTy.
1781 EVL = IB.CreateZExtOrTrunc(V: EVL, DestTy: IntptrTy);
1782 // To avoid undefined behavior for extracting with out of range index, use
1783 // the minimum of evl and element count as trip count.
1784 Value *EC = IB.CreateElementCount(Ty: IntptrTy, EC: VTy->getElementCount());
1785 EVL = IB.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: EVL, RHS: EC);
1786 } else {
1787 EVL = IB.CreateElementCount(Ty: IntptrTy, EC: VTy->getElementCount());
1788 }
1789
1790 // Cast Stride to IntptrTy.
1791 if (Stride)
1792 Stride = IB.CreateZExtOrTrunc(V: Stride, DestTy: IntptrTy);
1793
1794 SplitBlockAndInsertForEachLane(End: EVL, InsertBefore: LoopInsertBefore->getIterator(),
1795 Func: [&](IRBuilderBase &IRB, Value *Index) {
1796 Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx: Index);
1797 if (auto *MaskElemC = dyn_cast<ConstantInt>(Val: MaskElem)) {
1798 if (MaskElemC->isZero())
1799 // No check
1800 return;
1801 // Unconditional check
1802 } else {
1803 // Conditional check
1804 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
1805 Cond: MaskElem, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false);
1806 IRB.SetInsertPoint(ThenTerm);
1807 }
1808
1809 Value *InstrumentedAddress;
1810 if (isa<VectorType>(Val: Addr->getType())) {
1811 assert(
1812 cast<VectorType>(Addr->getType())->getElementType()->isPointerTy() &&
1813 "Expected vector of pointer.");
1814 InstrumentedAddress = IRB.CreateExtractElement(Vec: Addr, Idx: Index);
1815 } else if (Stride) {
1816 Index = IRB.CreateMul(LHS: Index, RHS: Stride);
1817 InstrumentedAddress = IRB.CreatePtrAdd(Ptr: Addr, Offset: Index);
1818 } else {
1819 InstrumentedAddress = IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, Index});
1820 }
1821 doInstrumentAddress(Pass, I, InsertBefore: &*IRB.GetInsertPoint(), Addr: InstrumentedAddress,
1822 Alignment, Granularity, TypeStoreSize: ElemTypeSize, IsWrite,
1823 SizeArgument, UseCalls, Exp, RTCI);
1824 });
1825}
1826
1827void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
1828 InterestingMemoryOperand &O, bool UseCalls,
1829 const DataLayout &DL,
1830 RuntimeCallInserter &RTCI) {
1831 Value *Addr = O.getPtr();
1832
1833 // Optimization experiments.
1834 // The experiments can be used to evaluate potential optimizations that remove
1835 // instrumentation (assess false negatives). Instead of completely removing
1836 // some instrumentation, you set Exp to a non-zero value (mask of optimization
1837 // experiments that want to remove instrumentation of this instruction).
1838 // If Exp is non-zero, this pass will emit special calls into runtime
1839 // (e.g. __asan_report_exp_load1 instead of __asan_report_load1). These calls
1840 // make runtime terminate the program in a special way (with a different
1841 // exit status). Then you run the new compiler on a buggy corpus, collect
1842 // the special terminations (ideally, you don't see them at all -- no false
1843 // negatives) and make the decision on the optimization.
1844 uint32_t Exp = ClForceExperiment;
1845
1846 if (ClOpt && ClOptGlobals) {
1847 // If initialization order checking is disabled, a simple access to a
1848 // dynamically initialized global is always valid.
1849 GlobalVariable *G = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: Addr));
1850 if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
1851 isSafeAccess(ObjSizeVis, Addr, TypeStoreSize: O.TypeStoreSize)) {
1852 NumOptimizedAccessesToGlobalVar++;
1853 return;
1854 }
1855 }
1856
1857 if (ClOpt && ClOptStack) {
1858 // A direct inbounds access to a stack variable is always valid.
1859 if (isa<AllocaInst>(Val: getUnderlyingObject(V: Addr)) &&
1860 isSafeAccess(ObjSizeVis, Addr, TypeStoreSize: O.TypeStoreSize)) {
1861 NumOptimizedAccessesToStackVar++;
1862 return;
1863 }
1864 }
1865
1866 if (O.IsWrite)
1867 NumInstrumentedWrites++;
1868 else
1869 NumInstrumentedReads++;
1870
1871 if (O.MaybeByteOffset) {
1872 Type *Ty = Type::getInt8Ty(C&: *C);
1873 IRBuilder IB(O.getInsn());
1874
1875 Value *OffsetOp = O.MaybeByteOffset;
1876 if (TargetTriple.isRISCV()) {
1877 Type *OffsetTy = OffsetOp->getType();
1878 // RVV indexed loads/stores zero-extend offset operands which are narrower
1879 // than XLEN to XLEN.
1880 if (OffsetTy->getScalarType()->getIntegerBitWidth() <
1881 static_cast<unsigned>(LongSize)) {
1882 VectorType *OrigType = cast<VectorType>(Val: OffsetTy);
1883 Type *ExtendTy = VectorType::get(ElementType: IntptrTy, Other: OrigType);
1884 OffsetOp = IB.CreateZExt(V: OffsetOp, DestTy: ExtendTy);
1885 }
1886 }
1887 Addr = IB.CreateGEP(Ty, Ptr: Addr, IdxList: {OffsetOp});
1888 }
1889
1890 unsigned Granularity = 1 << Mapping.Scale;
1891 if (O.MaybeMask) {
1892 instrumentMaskedLoadOrStore(Pass: this, DL, IntptrTy, Mask: O.MaybeMask, EVL: O.MaybeEVL,
1893 Stride: O.MaybeStride, I: O.getInsn(), Addr, Alignment: O.Alignment,
1894 Granularity, OpType: O.OpType, IsWrite: O.IsWrite, SizeArgument: nullptr,
1895 UseCalls, Exp, RTCI);
1896 } else {
1897 doInstrumentAddress(Pass: this, I: O.getInsn(), InsertBefore: O.getInsn(), Addr, Alignment: O.Alignment,
1898 Granularity, TypeStoreSize: O.TypeStoreSize, IsWrite: O.IsWrite, SizeArgument: nullptr,
1899 UseCalls, Exp, RTCI);
1900 }
1901}
1902
1903Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore,
1904 Value *Addr, bool IsWrite,
1905 size_t AccessSizeIndex,
1906 Value *SizeArgument,
1907 uint32_t Exp,
1908 RuntimeCallInserter &RTCI) {
1909 InstrumentationIRBuilder IRB(InsertBefore);
1910 Value *ExpVal = Exp == 0 ? nullptr : ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp);
1911 CallInst *Call = nullptr;
1912 if (SizeArgument) {
1913 if (Exp == 0)
1914 Call = RTCI.createRuntimeCall(IRB, Callee: AsanErrorCallbackSized[IsWrite][0],
1915 Args: {Addr, SizeArgument});
1916 else
1917 Call = RTCI.createRuntimeCall(IRB, Callee: AsanErrorCallbackSized[IsWrite][1],
1918 Args: {Addr, SizeArgument, ExpVal});
1919 } else {
1920 if (Exp == 0)
1921 Call = RTCI.createRuntimeCall(
1922 IRB, Callee: AsanErrorCallback[IsWrite][0][AccessSizeIndex], Args: Addr);
1923 else
1924 Call = RTCI.createRuntimeCall(
1925 IRB, Callee: AsanErrorCallback[IsWrite][1][AccessSizeIndex], Args: {Addr, ExpVal});
1926 }
1927
1928 Call->setCannotMerge();
1929 return Call;
1930}
1931
1932Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
1933 Value *ShadowValue,
1934 uint32_t TypeStoreSize) {
1935 size_t Granularity = static_cast<size_t>(1) << Mapping.Scale;
1936 // Addr & (Granularity - 1)
1937 Value *LastAccessedByte =
1938 IRB.CreateAnd(LHS: AddrLong, RHS: ConstantInt::get(Ty: IntptrTy, V: Granularity - 1));
1939 // (Addr & (Granularity - 1)) + size - 1
1940 if (TypeStoreSize / 8 > 1)
1941 LastAccessedByte = IRB.CreateAdd(
1942 LHS: LastAccessedByte, RHS: ConstantInt::get(Ty: IntptrTy, V: TypeStoreSize / 8 - 1));
1943 // (uint8_t) ((Addr & (Granularity-1)) + size - 1)
1944 LastAccessedByte =
1945 IRB.CreateIntCast(V: LastAccessedByte, DestTy: ShadowValue->getType(), isSigned: false);
1946 // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
1947 return IRB.CreateICmpSGE(LHS: LastAccessedByte, RHS: ShadowValue);
1948}
1949
1950Instruction *AddressSanitizer::instrumentAMDGPUAddress(
1951 Instruction *OrigIns, Instruction *InsertBefore, Value *Addr,
1952 uint32_t TypeStoreSize, bool IsWrite, Value *SizeArgument) {
1953 // Do not instrument unsupported addrspaces.
1954 if (isUnsupportedAMDGPUAddrspace(Addr))
1955 return nullptr;
1956 Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
1957 // Follow host instrumentation for global and constant addresses.
1958 if (PtrTy->getPointerAddressSpace() != 0)
1959 return InsertBefore;
1960 // Instrument generic addresses in supported addressspaces.
1961 IRBuilder<> IRB(InsertBefore);
1962 Value *IsShared = IRB.CreateCall(Callee: AMDGPUAddressShared, Args: {Addr});
1963 Value *IsPrivate = IRB.CreateCall(Callee: AMDGPUAddressPrivate, Args: {Addr});
1964 Value *IsSharedOrPrivate = IRB.CreateOr(LHS: IsShared, RHS: IsPrivate);
1965 Value *Cmp = IRB.CreateNot(V: IsSharedOrPrivate);
1966 Value *AddrSpaceZeroLanding =
1967 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, Unreachable: false);
1968 InsertBefore = cast<Instruction>(Val: AddrSpaceZeroLanding);
1969 return InsertBefore;
1970}
1971
1972Instruction *AddressSanitizer::genAMDGPUReportBlock(IRBuilder<> &IRB,
1973 Value *Cond, bool Recover) {
1974 Value *ReportCond = Cond;
1975 if (!Recover) {
1976 auto Ballot = Inserter.insertFunction(Name: kAMDGPUBallotName, Args: IRB.getInt64Ty(),
1977 Args: IRB.getInt1Ty());
1978 ReportCond = IRB.CreateIsNotNull(Arg: IRB.CreateCall(Callee: Ballot, Args: {Cond}));
1979 }
1980
1981 auto *Trm =
1982 SplitBlockAndInsertIfThen(Cond: ReportCond, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false,
1983 BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights());
1984 Trm->getParent()->setName("asan.report");
1985
1986 if (Recover)
1987 return Trm;
1988
1989 Trm = SplitBlockAndInsertIfThen(Cond, SplitBefore: Trm, Unreachable: false);
1990 IRB.SetInsertPoint(Trm);
1991 return IRB.CreateCall(
1992 Callee: Inserter.insertFunction(Name: kAMDGPUUnreachableName, Args: IRB.getVoidTy()), Args: {});
1993}
1994
1995void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
1996 Instruction *InsertBefore, Value *Addr,
1997 MaybeAlign Alignment,
1998 uint32_t TypeStoreSize, bool IsWrite,
1999 Value *SizeArgument, bool UseCalls,
2000 uint32_t Exp,
2001 RuntimeCallInserter &RTCI) {
2002 if (TargetTriple.isAMDGPU()) {
2003 InsertBefore = instrumentAMDGPUAddress(OrigIns, InsertBefore, Addr,
2004 TypeStoreSize, IsWrite, SizeArgument);
2005 if (!InsertBefore)
2006 return;
2007 }
2008
2009 InstrumentationIRBuilder IRB(InsertBefore);
2010 size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeSize: TypeStoreSize);
2011
2012 if (UseCalls && ClOptimizeCallbacks) {
2013 const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
2014 IRB.CreateIntrinsic(ID: Intrinsic::asan_check_memaccess, OverloadTypes: {},
2015 Args: {IRB.CreatePointerCast(V: Addr, DestTy: PtrTy),
2016 ConstantInt::get(Ty: Int32Ty, V: AccessInfo.Packed)});
2017 return;
2018 }
2019
2020 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
2021 if (UseCalls) {
2022 if (Exp == 0)
2023 RTCI.createRuntimeCall(
2024 IRB, Callee: AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex], Args: AddrLong);
2025 else
2026 RTCI.createRuntimeCall(
2027 IRB, Callee: AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex],
2028 Args: {AddrLong, ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp)});
2029 return;
2030 }
2031
2032 Type *ShadowTy =
2033 IntegerType::get(C&: *C, NumBits: std::max(a: 8U, b: TypeStoreSize >> Mapping.Scale));
2034 Type *ShadowPtrTy = PointerType::get(C&: *C, AddressSpace: ClShadowAddrSpace);
2035 Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
2036 const uint64_t ShadowAlign =
2037 std::max<uint64_t>(a: Alignment.valueOrOne().value() >> Mapping.Scale, b: 1);
2038 Value *ShadowValue = IRB.CreateAlignedLoad(
2039 Ty: ShadowTy, Ptr: IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy), Align: Align(ShadowAlign));
2040
2041 Value *Cmp = IRB.CreateIsNotNull(Arg: ShadowValue);
2042 size_t Granularity = 1ULL << Mapping.Scale;
2043 Instruction *CrashTerm = nullptr;
2044
2045 bool GenSlowPath = (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity));
2046
2047 if (TargetTriple.isAMDGCN()) {
2048 if (GenSlowPath) {
2049 auto *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
2050 Cmp = IRB.CreateAnd(LHS: Cmp, RHS: Cmp2);
2051 }
2052 CrashTerm = genAMDGPUReportBlock(IRB, Cond: Cmp, Recover);
2053 } else if (GenSlowPath) {
2054 // We use branch weights for the slow path check, to indicate that the slow
2055 // path is rarely taken. This seems to be the case for SPEC benchmarks.
2056 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
2057 Cond: Cmp, SplitBefore: InsertBefore, Unreachable: false, BranchWeights: MDBuilder(*C).createUnlikelyBranchWeights());
2058 BasicBlock *NextBB = cast<UncondBrInst>(Val: CheckTerm)->getSuccessor();
2059 IRB.SetInsertPoint(CheckTerm);
2060 Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
2061 if (Recover) {
2062 CrashTerm = SplitBlockAndInsertIfThen(Cond: Cmp2, SplitBefore: CheckTerm, Unreachable: false);
2063 } else {
2064 BasicBlock *CrashBlock =
2065 BasicBlock::Create(Context&: *C, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
2066 CrashTerm = new UnreachableInst(*C, CrashBlock);
2067 CondBrInst *NewTerm = CondBrInst::Create(Cond: Cmp2, IfTrue: CrashBlock, IfFalse: NextBB);
2068 ReplaceInstWithInst(From: CheckTerm, To: NewTerm);
2069 }
2070 } else {
2071 CrashTerm = SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, Unreachable: !Recover);
2072 }
2073
2074 Instruction *Crash = generateCrashCode(
2075 InsertBefore: CrashTerm, Addr: AddrLong, IsWrite, AccessSizeIndex, SizeArgument, Exp, RTCI);
2076 if (OrigIns->getDebugLoc())
2077 Crash->setDebugLoc(OrigIns->getDebugLoc());
2078}
2079
2080// Instrument unusual size or unusual alignment.
2081// We can not do it with a single check, so we do 1-byte check for the first
2082// and the last bytes. We call __asan_report_*_n(addr, real_size) to be able
2083// to report the actual access size.
2084void AddressSanitizer::instrumentUnusualSizeOrAlignment(
2085 Instruction *I, Instruction *InsertBefore, Value *Addr,
2086 TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls,
2087 uint32_t Exp, RuntimeCallInserter &RTCI) {
2088 InstrumentationIRBuilder IRB(InsertBefore);
2089 Value *NumBits = IRB.CreateTypeSize(Ty: IntptrTy, Size: TypeStoreSize);
2090 Value *Size = IRB.CreateLShr(LHS: NumBits, RHS: ConstantInt::get(Ty: IntptrTy, V: 3));
2091
2092 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
2093 if (UseCalls) {
2094 if (Exp == 0)
2095 RTCI.createRuntimeCall(IRB, Callee: AsanMemoryAccessCallbackSized[IsWrite][0],
2096 Args: {AddrLong, Size});
2097 else
2098 RTCI.createRuntimeCall(
2099 IRB, Callee: AsanMemoryAccessCallbackSized[IsWrite][1],
2100 Args: {AddrLong, Size, ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp)});
2101 } else {
2102 Value *SizeMinusOne = IRB.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: IntptrTy, V: 1));
2103 Value *LastByte = IRB.CreateIntToPtr(
2104 V: IRB.CreateAdd(LHS: AddrLong, RHS: SizeMinusOne),
2105 DestTy: Addr->getType());
2106 instrumentAddress(OrigIns: I, InsertBefore, Addr, Alignment: {}, TypeStoreSize: 8, IsWrite, SizeArgument: Size, UseCalls: false, Exp,
2107 RTCI);
2108 instrumentAddress(OrigIns: I, InsertBefore, Addr: LastByte, Alignment: {}, TypeStoreSize: 8, IsWrite, SizeArgument: Size, UseCalls: false,
2109 Exp, RTCI);
2110 }
2111}
2112
2113void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit) {
2114 // Set up the arguments to our poison/unpoison functions.
2115 IRBuilder<> IRB(GlobalInit.front().getFirstInsertionPt());
2116
2117 // Add a call to poison all external globals before the given function starts.
2118 Value *ModuleNameAddr =
2119 ConstantExpr::getPointerCast(C: getOrCreateModuleName(), Ty: IntptrTy);
2120 CallInst *CallBefore = IRB.CreateCall(Callee: AsanPoisonGlobals, Args: ModuleNameAddr);
2121 if (DISubprogram *SP = GlobalInit.getSubprogram())
2122 CallBefore->setDebugLoc(
2123 DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP));
2124
2125 // Add calls to unpoison all globals before each return instruction.
2126 for (auto &BB : GlobalInit)
2127 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator())) {
2128 CallInst *CallAfter =
2129 CallInst::Create(Func: AsanUnpoisonGlobals, NameStr: "", InsertBefore: RI->getIterator());
2130 if (RI->getDebugLoc())
2131 CallAfter->setDebugLoc(RI->getDebugLoc());
2132 else if (DISubprogram *SP = GlobalInit.getSubprogram())
2133 CallAfter->setDebugLoc(
2134 DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP));
2135 }
2136}
2137
2138void ModuleAddressSanitizer::createInitializerPoisonCalls() {
2139 GlobalVariable *GV = M.getGlobalVariable(Name: "llvm.global_ctors");
2140 if (!GV)
2141 return;
2142
2143 ConstantArray *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
2144 if (!CA)
2145 return;
2146
2147 for (Use &OP : CA->operands()) {
2148 if (isa<ConstantAggregateZero>(Val: OP)) continue;
2149 ConstantStruct *CS = cast<ConstantStruct>(Val&: OP);
2150
2151 // Must have a function or null ptr.
2152 if (Function *F = dyn_cast<Function>(Val: CS->getOperand(i_nocapture: 1))) {
2153 if (F->getName() == kAsanModuleCtorName) continue;
2154 auto *Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0));
2155 // Don't instrument CTORs that will run before asan.module_ctor.
2156 if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
2157 continue;
2158 poisonOneInitializer(GlobalInit&: *F);
2159 }
2160 }
2161}
2162
2163const GlobalVariable *
2164ModuleAddressSanitizer::getExcludedAliasedGlobal(const GlobalAlias &GA) const {
2165 // In case this function should be expanded to include rules that do not just
2166 // apply when CompileKernel is true, either guard all existing rules with an
2167 // 'if (CompileKernel) { ... }' or be absolutely sure that all these rules
2168 // should also apply to user space.
2169 assert(CompileKernel && "Only expecting to be called when compiling kernel");
2170
2171 const Constant *C = GA.getAliasee();
2172
2173 // When compiling the kernel, globals that are aliased by symbols prefixed
2174 // by "__" are special and cannot be padded with a redzone.
2175 if (GA.getName().starts_with(Prefix: "__"))
2176 return dyn_cast<GlobalVariable>(Val: C->stripPointerCastsAndAliases());
2177
2178 return nullptr;
2179}
2180
2181bool ModuleAddressSanitizer::shouldInstrumentGlobal(GlobalVariable *G) const {
2182 Type *Ty = G->getValueType();
2183 LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
2184
2185 if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().NoAddress)
2186 return false;
2187 if (!Ty->isSized()) return false;
2188 if (!G->hasInitializer()) return false;
2189 if (!isSupportedAddrspace(TargetTriple, Addr: G))
2190 return false;
2191 if (GlobalWasGeneratedByCompiler(G)) return false; // Our own globals.
2192 // Two problems with thread-locals:
2193 // - The address of the main thread's copy can't be computed at link-time.
2194 // - Need to poison all copies, not just the main thread's one.
2195 if (G->isThreadLocal()) return false;
2196 // For now, just ignore this Global if the alignment is large.
2197 if (G->getAlign() && *G->getAlign() > getMinRedzoneSizeForGlobal()) return false;
2198
2199 // For non-COFF targets, only instrument globals known to be defined by this
2200 // TU.
2201 // FIXME: We can instrument comdat globals on ELF if we are using the
2202 // GC-friendly metadata scheme.
2203 if (!TargetTriple.isOSBinFormatCOFF()) {
2204 if (!G->hasExactDefinition() || G->hasComdat())
2205 return false;
2206 } else {
2207 // On COFF, don't instrument non-ODR linkages.
2208 if (G->isInterposable())
2209 return false;
2210 // If the global has AvailableExternally linkage, then it is not in this
2211 // module, which means it does not need to be instrumented.
2212 if (G->hasAvailableExternallyLinkage())
2213 return false;
2214 }
2215
2216 // If a comdat is present, it must have a selection kind that implies ODR
2217 // semantics: no duplicates, any, or exact match.
2218 if (Comdat *C = G->getComdat()) {
2219 switch (C->getSelectionKind()) {
2220 case Comdat::Any:
2221 case Comdat::ExactMatch:
2222 case Comdat::NoDeduplicate:
2223 break;
2224 case Comdat::Largest:
2225 case Comdat::SameSize:
2226 return false;
2227 }
2228 }
2229
2230 if (G->hasSection()) {
2231 // The kernel uses explicit sections for mostly special global variables
2232 // that we should not instrument. E.g. the kernel may rely on their layout
2233 // without redzones, or remove them at link time ("discard.*"), etc.
2234 if (CompileKernel)
2235 return false;
2236
2237 StringRef Section = G->getSection();
2238
2239 // Globals from llvm.metadata aren't emitted, do not instrument them.
2240 if (Section == "llvm.metadata") return false;
2241 // Do not instrument globals from special LLVM sections.
2242 if (Section.contains(Other: "__llvm") || Section.contains(Other: "__LLVM"))
2243 return false;
2244
2245 // Do not instrument function pointers to initialization and termination
2246 // routines: dynamic linker will not properly handle redzones.
2247 if (Section.starts_with(Prefix: ".preinit_array") ||
2248 Section.starts_with(Prefix: ".init_array") ||
2249 Section.starts_with(Prefix: ".fini_array")) {
2250 return false;
2251 }
2252
2253 // Do not instrument user-defined sections (with names resembling
2254 // valid C identifiers)
2255 if (TargetTriple.isOSBinFormatELF()) {
2256 if (llvm::all_of(Range&: Section,
2257 P: [](char c) { return llvm::isAlnum(C: c) || c == '_'; }))
2258 return false;
2259 }
2260
2261 // On COFF, if the section name contains '$', it is highly likely that the
2262 // user is using section sorting to create an array of globals similar to
2263 // the way initialization callbacks are registered in .init_array and
2264 // .CRT$XCU. The ATL also registers things in .ATL$__[azm]. Adding redzones
2265 // to such globals is counterproductive, because the intent is that they
2266 // will form an array, and out-of-bounds accesses are expected.
2267 // See https://github.com/google/sanitizers/issues/305
2268 // and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
2269 if (TargetTriple.isOSBinFormatCOFF() && Section.contains(C: '$')) {
2270 LLVM_DEBUG(dbgs() << "Ignoring global in sorted section (contains '$'): "
2271 << *G << "\n");
2272 return false;
2273 }
2274
2275 if (TargetTriple.isOSBinFormatMachO()) {
2276 StringRef ParsedSegment, ParsedSection;
2277 unsigned TAA = 0, StubSize = 0;
2278 bool TAAParsed;
2279 cantFail(Err: MCSectionMachO::ParseSectionSpecifier(
2280 Spec: Section, Segment&: ParsedSegment, Section&: ParsedSection, TAA, TAAParsed, StubSize));
2281
2282 // Ignore the globals from the __OBJC section. The ObjC runtime assumes
2283 // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
2284 // them.
2285 if (ParsedSegment == "__OBJC" ||
2286 (ParsedSegment == "__DATA" && ParsedSection.starts_with(Prefix: "__objc_"))) {
2287 LLVM_DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
2288 return false;
2289 }
2290 // See https://github.com/google/sanitizers/issues/32
2291 // Constant CFString instances are compiled in the following way:
2292 // -- the string buffer is emitted into
2293 // __TEXT,__cstring,cstring_literals
2294 // -- the constant NSConstantString structure referencing that buffer
2295 // is placed into __DATA,__cfstring
2296 // Therefore there's no point in placing redzones into __DATA,__cfstring.
2297 // Moreover, it causes the linker to crash on OS X 10.7
2298 if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") {
2299 LLVM_DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
2300 return false;
2301 }
2302 // The linker merges the contents of cstring_literals and removes the
2303 // trailing zeroes.
2304 if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) {
2305 LLVM_DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
2306 return false;
2307 }
2308 }
2309 }
2310
2311 if (CompileKernel) {
2312 // Globals that prefixed by "__" are special and cannot be padded with a
2313 // redzone.
2314 if (G->getName().starts_with(Prefix: "__"))
2315 return false;
2316 }
2317
2318 return true;
2319}
2320
2321// On Mach-O platforms, we emit global metadata in a separate section of the
2322// binary in order to allow the linker to properly dead strip. This is only
2323// supported on recent versions of ld64.
2324bool ModuleAddressSanitizer::ShouldUseMachOGlobalsSection() const {
2325 if (!TargetTriple.isOSBinFormatMachO())
2326 return false;
2327
2328 if (TargetTriple.isMacOSX() && !TargetTriple.isMacOSXVersionLT(Major: 10, Minor: 11))
2329 return true;
2330 if (TargetTriple.isiOS() /* or tvOS */ && !TargetTriple.isOSVersionLT(Major: 9))
2331 return true;
2332 if (TargetTriple.isWatchOS() && !TargetTriple.isOSVersionLT(Major: 2))
2333 return true;
2334 if (TargetTriple.isDriverKit())
2335 return true;
2336 if (TargetTriple.isXROS())
2337 return true;
2338
2339 return false;
2340}
2341
2342StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
2343 switch (TargetTriple.getObjectFormat()) {
2344 case Triple::COFF: return ".ASAN$GL";
2345 case Triple::ELF: return "asan_globals";
2346 case Triple::MachO: return "__DATA,__asan_globals,regular";
2347 case Triple::Wasm:
2348 case Triple::GOFF:
2349 case Triple::SPIRV:
2350 case Triple::XCOFF:
2351 case Triple::DXContainer:
2352 report_fatal_error(
2353 reason: "ModuleAddressSanitizer not implemented for object file format");
2354 case Triple::UnknownObjectFormat:
2355 break;
2356 }
2357 llvm_unreachable("unsupported object format");
2358}
2359
2360void ModuleAddressSanitizer::initializeCallbacks() {
2361 IRBuilder<> IRB(M);
2362
2363 // Declare our poisoning and unpoisoning functions.
2364 AsanPoisonGlobals = Inserter.insertFunction(Name: kAsanPoisonGlobalsName,
2365 Args: IRB.getVoidTy(), Args&: IntptrTy);
2366 AsanUnpoisonGlobals =
2367 Inserter.insertFunction(Name: kAsanUnpoisonGlobalsName, Args: IRB.getVoidTy());
2368
2369 // Declare functions that register/unregister globals.
2370 AsanRegisterGlobals = Inserter.insertFunction(
2371 Name: kAsanRegisterGlobalsName, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
2372 AsanUnregisterGlobals = Inserter.insertFunction(
2373 Name: kAsanUnregisterGlobalsName, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
2374
2375 // Declare the functions that find globals in a shared object and then invoke
2376 // the (un)register function on them.
2377 AsanRegisterImageGlobals = Inserter.insertFunction(
2378 Name: kAsanRegisterImageGlobalsName, Args: IRB.getVoidTy(), Args&: IntptrTy);
2379 AsanUnregisterImageGlobals = Inserter.insertFunction(
2380 Name: kAsanUnregisterImageGlobalsName, Args: IRB.getVoidTy(), Args&: IntptrTy);
2381
2382 AsanRegisterElfGlobals =
2383 Inserter.insertFunction(Name: kAsanRegisterElfGlobalsName, Args: IRB.getVoidTy(),
2384 Args&: IntptrTy, Args&: IntptrTy, Args&: IntptrTy);
2385 AsanUnregisterElfGlobals =
2386 Inserter.insertFunction(Name: kAsanUnregisterElfGlobalsName, Args: IRB.getVoidTy(),
2387 Args&: IntptrTy, Args&: IntptrTy, Args&: IntptrTy);
2388}
2389
2390// Put the metadata and the instrumented global in the same group. This ensures
2391// that the metadata is discarded if the instrumented global is discarded.
2392void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
2393 GlobalVariable *G, GlobalVariable *Metadata, StringRef InternalSuffix) {
2394 Module &M = *G->getParent();
2395 Comdat *C = G->getComdat();
2396 if (!C) {
2397 if (!G->hasName()) {
2398 // If G is unnamed, it must be internal. Give it an artificial name
2399 // so we can put it in a comdat.
2400 assert(G->hasLocalLinkage());
2401 G->setName(genName(suffix: "anon_global"));
2402 }
2403
2404 if (!InternalSuffix.empty() && G->hasLocalLinkage()) {
2405 std::string Name = std::string(G->getName());
2406 Name += InternalSuffix;
2407 C = M.getOrInsertComdat(Name);
2408 } else {
2409 C = M.getOrInsertComdat(Name: G->getName());
2410 }
2411
2412 // Make this IMAGE_COMDAT_SELECT_NODUPLICATES on COFF. Also upgrade private
2413 // linkage to internal linkage so that a symbol table entry is emitted. This
2414 // is necessary in order to create the comdat group.
2415 if (TargetTriple.isOSBinFormatCOFF()) {
2416 C->setSelectionKind(Comdat::NoDeduplicate);
2417 if (G->hasPrivateLinkage())
2418 G->setLinkage(GlobalValue::InternalLinkage);
2419 }
2420 G->setComdat(C);
2421 }
2422
2423 assert(G->hasComdat());
2424 Metadata->setComdat(G->getComdat());
2425}
2426
2427// Create a separate metadata global and put it in the appropriate ASan
2428// global registration section.
2429GlobalVariable *
2430ModuleAddressSanitizer::CreateMetadataGlobal(Constant *Initializer,
2431 StringRef OriginalName) {
2432 auto Linkage = TargetTriple.isOSBinFormatMachO()
2433 ? GlobalVariable::InternalLinkage
2434 : GlobalVariable::PrivateLinkage;
2435 GlobalVariable *Metadata = new GlobalVariable(
2436 M, Initializer->getType(), false, Linkage, Initializer,
2437 Twine("__asan_global_") + GlobalValue::dropLLVMManglingEscape(Name: OriginalName));
2438 Metadata->setSection(getGlobalMetadataSection());
2439 // Place metadata in a large section for x86-64 ELF binaries to mitigate
2440 // relocation pressure.
2441 setGlobalVariableLargeSection(TargetTriple, GV&: *Metadata);
2442 return Metadata;
2443}
2444
2445Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor() {
2446 AsanDtorFunction = Function::createWithDefaultAttr(
2447 Ty: FunctionType::get(Result: Type::getVoidTy(C&: *C), isVarArg: false),
2448 Linkage: GlobalValue::InternalLinkage, AddrSpace: 0, N: kAsanModuleDtorName, M: &M);
2449 AsanDtorFunction->addFnAttr(Kind: Attribute::NoUnwind);
2450 // Ensure Dtor cannot be discarded, even if in a comdat.
2451 appendToUsed(M, Values: {AsanDtorFunction});
2452 BasicBlock *AsanDtorBB = BasicBlock::Create(Context&: *C, Name: "", Parent: AsanDtorFunction);
2453
2454 return ReturnInst::Create(C&: *C, InsertAtEnd: AsanDtorBB);
2455}
2456
2457void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
2458 IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
2459 ArrayRef<Constant *> MetadataInitializers) {
2460 assert(ExtendedGlobals.size() == MetadataInitializers.size());
2461 auto &DL = M.getDataLayout();
2462
2463 SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
2464 for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2465 Constant *Initializer = MetadataInitializers[i];
2466 GlobalVariable *G = ExtendedGlobals[i];
2467 GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, OriginalName: G->getName());
2468 MDNode *MD = MDNode::get(Context&: M.getContext(), MDs: ValueAsMetadata::get(V: G));
2469 Metadata->setMetadata(KindID: LLVMContext::MD_associated, Node: MD);
2470 MetadataGlobals[i] = Metadata;
2471
2472 // The MSVC linker always inserts padding when linking incrementally. We
2473 // cope with that by aligning each struct to its size, which must be a power
2474 // of two.
2475 unsigned SizeOfGlobalStruct = DL.getTypeAllocSize(Ty: Initializer->getType());
2476 assert(isPowerOf2_32(SizeOfGlobalStruct) &&
2477 "global metadata will not be padded appropriately");
2478 Metadata->setAlignment(assumeAligned(Value: SizeOfGlobalStruct));
2479
2480 SetComdatForGlobalMetadata(G, Metadata, InternalSuffix: "");
2481 }
2482
2483 // Update llvm.compiler.used, adding the new metadata globals. This is
2484 // needed so that during LTO these variables stay alive.
2485 if (!MetadataGlobals.empty())
2486 appendToCompilerUsed(M, Values: MetadataGlobals);
2487}
2488
2489void ModuleAddressSanitizer::instrumentGlobalsELF(
2490 IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
2491 ArrayRef<Constant *> MetadataInitializers,
2492 const std::string &UniqueModuleId) {
2493 assert(ExtendedGlobals.size() == MetadataInitializers.size());
2494
2495 // Putting globals in a comdat changes the semantic and potentially cause
2496 // false negative odr violations at link time. If odr indicators are used, we
2497 // keep the comdat sections, as link time odr violations will be detected on
2498 // the odr indicator symbols.
2499 bool UseComdatForGlobalsGC = UseOdrIndicator && !UniqueModuleId.empty();
2500
2501 SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
2502 for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2503 GlobalVariable *G = ExtendedGlobals[i];
2504 GlobalVariable *Metadata =
2505 CreateMetadataGlobal(Initializer: MetadataInitializers[i], OriginalName: G->getName());
2506 MDNode *MD = MDNode::get(Context&: M.getContext(), MDs: ValueAsMetadata::get(V: G));
2507 Metadata->setMetadata(KindID: LLVMContext::MD_associated, Node: MD);
2508 MetadataGlobals[i] = Metadata;
2509
2510 if (UseComdatForGlobalsGC)
2511 SetComdatForGlobalMetadata(G, Metadata, InternalSuffix: UniqueModuleId);
2512 }
2513
2514 // Update llvm.compiler.used, adding the new metadata globals. This is
2515 // needed so that during LTO these variables stay alive.
2516 if (!MetadataGlobals.empty())
2517 appendToCompilerUsed(M, Values: MetadataGlobals);
2518
2519 // RegisteredFlag serves two purposes. First, we can pass it to dladdr()
2520 // to look up the loaded image that contains it. Second, we can store in it
2521 // whether registration has already occurred, to prevent duplicate
2522 // registration.
2523 //
2524 // Common linkage ensures that there is only one global per shared library.
2525 GlobalVariable *RegisteredFlag = new GlobalVariable(
2526 M, IntptrTy, false, GlobalVariable::CommonLinkage,
2527 ConstantInt::get(Ty: IntptrTy, V: 0), kAsanGlobalsRegisteredFlagName);
2528 RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
2529
2530 // Create start and stop symbols.
2531 GlobalVariable *StartELFMetadata = new GlobalVariable(
2532 M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2533 "__start_" + getGlobalMetadataSection());
2534 StartELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
2535 GlobalVariable *StopELFMetadata = new GlobalVariable(
2536 M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2537 "__stop_" + getGlobalMetadataSection());
2538 StopELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
2539
2540 // Create a call to register the globals with the runtime.
2541 if (ConstructorKind == AsanCtorKind::Global)
2542 IRB.CreateCall(Callee: AsanRegisterElfGlobals,
2543 Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy),
2544 IRB.CreatePointerCast(V: StartELFMetadata, DestTy: IntptrTy),
2545 IRB.CreatePointerCast(V: StopELFMetadata, DestTy: IntptrTy)});
2546
2547 // We also need to unregister globals at the end, e.g., when a shared library
2548 // gets closed.
2549 if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
2550 IRBuilder<> IrbDtor(CreateAsanModuleDtor());
2551 IrbDtor.CreateCall(Callee: AsanUnregisterElfGlobals,
2552 Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy),
2553 IRB.CreatePointerCast(V: StartELFMetadata, DestTy: IntptrTy),
2554 IRB.CreatePointerCast(V: StopELFMetadata, DestTy: IntptrTy)});
2555 }
2556}
2557
2558void ModuleAddressSanitizer::InstrumentGlobalsMachO(
2559 IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
2560 ArrayRef<Constant *> MetadataInitializers) {
2561 assert(ExtendedGlobals.size() == MetadataInitializers.size());
2562
2563 // On recent Mach-O platforms, use a structure which binds the liveness of
2564 // the global variable to the metadata struct. Keep the list of "Liveness" GV
2565 // created to be added to llvm.compiler.used
2566 StructType *LivenessTy = StructType::get(elt1: IntptrTy, elts: IntptrTy);
2567 SmallVector<GlobalValue *, 16> LivenessGlobals(ExtendedGlobals.size());
2568
2569 for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2570 Constant *Initializer = MetadataInitializers[i];
2571 GlobalVariable *G = ExtendedGlobals[i];
2572 GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, OriginalName: G->getName());
2573
2574 // On recent Mach-O platforms, we emit the global metadata in a way that
2575 // allows the linker to properly strip dead globals.
2576 auto LivenessBinder =
2577 ConstantStruct::get(T: LivenessTy, Vs: Initializer->getAggregateElement(Elt: 0u),
2578 Vs: ConstantExpr::getPointerCast(C: Metadata, Ty: IntptrTy));
2579 GlobalVariable *Liveness = new GlobalVariable(
2580 M, LivenessTy, false, GlobalVariable::InternalLinkage, LivenessBinder,
2581 Twine("__asan_binder_") + G->getName());
2582 Liveness->setSection("__DATA,__asan_liveness,regular,live_support");
2583 LivenessGlobals[i] = Liveness;
2584 }
2585
2586 // Update llvm.compiler.used, adding the new liveness globals. This is
2587 // needed so that during LTO these variables stay alive. The alternative
2588 // would be to have the linker handling the LTO symbols, but libLTO
2589 // current API does not expose access to the section for each symbol.
2590 if (!LivenessGlobals.empty())
2591 appendToCompilerUsed(M, Values: LivenessGlobals);
2592
2593 // RegisteredFlag serves two purposes. First, we can pass it to dladdr()
2594 // to look up the loaded image that contains it. Second, we can store in it
2595 // whether registration has already occurred, to prevent duplicate
2596 // registration.
2597 //
2598 // common linkage ensures that there is only one global per shared library.
2599 GlobalVariable *RegisteredFlag = new GlobalVariable(
2600 M, IntptrTy, false, GlobalVariable::CommonLinkage,
2601 ConstantInt::get(Ty: IntptrTy, V: 0), kAsanGlobalsRegisteredFlagName);
2602 RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
2603
2604 if (ConstructorKind == AsanCtorKind::Global)
2605 IRB.CreateCall(Callee: AsanRegisterImageGlobals,
2606 Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy)});
2607
2608 // We also need to unregister globals at the end, e.g., when a shared library
2609 // gets closed.
2610 if (DestructorKind != AsanDtorKind::None) {
2611 IRBuilder<> IrbDtor(CreateAsanModuleDtor());
2612 IrbDtor.CreateCall(Callee: AsanUnregisterImageGlobals,
2613 Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy)});
2614 }
2615}
2616
2617void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
2618 IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
2619 ArrayRef<Constant *> MetadataInitializers) {
2620 assert(ExtendedGlobals.size() == MetadataInitializers.size());
2621 unsigned N = ExtendedGlobals.size();
2622 assert(N > 0);
2623
2624 // On platforms that don't have a custom metadata section, we emit an array
2625 // of global metadata structures.
2626 ArrayType *ArrayOfGlobalStructTy =
2627 ArrayType::get(ElementType: MetadataInitializers[0]->getType(), NumElements: N);
2628 auto AllGlobals = new GlobalVariable(
2629 M, ArrayOfGlobalStructTy, false, GlobalVariable::InternalLinkage,
2630 ConstantArray::get(T: ArrayOfGlobalStructTy, V: MetadataInitializers), "");
2631 if (Mapping.Scale > 3)
2632 AllGlobals->setAlignment(Align(1ULL << Mapping.Scale));
2633
2634 if (ConstructorKind == AsanCtorKind::Global)
2635 IRB.CreateCall(Callee: AsanRegisterGlobals,
2636 Args: {IRB.CreatePointerCast(V: AllGlobals, DestTy: IntptrTy),
2637 ConstantInt::get(Ty: IntptrTy, V: N)});
2638
2639 // We also need to unregister globals at the end, e.g., when a shared library
2640 // gets closed.
2641 if (DestructorKind != AsanDtorKind::None) {
2642 IRBuilder<> IrbDtor(CreateAsanModuleDtor());
2643 IrbDtor.CreateCall(Callee: AsanUnregisterGlobals,
2644 Args: {IRB.CreatePointerCast(V: AllGlobals, DestTy: IntptrTy),
2645 ConstantInt::get(Ty: IntptrTy, V: N)});
2646 }
2647}
2648
2649// This function replaces all global variables with new variables that have
2650// trailing redzones. It also creates a function that poisons
2651// redzones and inserts this function into llvm.global_ctors.
2652// Sets *CtorComdat to true if the global registration code emitted into the
2653// asan constructor is comdat-compatible.
2654void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
2655 bool *CtorComdat) {
2656 // Build set of globals that are aliased by some GA, where
2657 // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
2658 SmallPtrSet<const GlobalVariable *, 16> AliasedGlobalExclusions;
2659 if (CompileKernel) {
2660 for (auto &GA : M.aliases()) {
2661 if (const GlobalVariable *GV = getExcludedAliasedGlobal(GA))
2662 AliasedGlobalExclusions.insert(Ptr: GV);
2663 }
2664 }
2665
2666 SmallVector<GlobalVariable *, 16> GlobalsToChange;
2667 for (auto &G : M.globals()) {
2668 if (!AliasedGlobalExclusions.count(Ptr: &G) && shouldInstrumentGlobal(G: &G))
2669 GlobalsToChange.push_back(Elt: &G);
2670 }
2671
2672 size_t n = GlobalsToChange.size();
2673 auto &DL = M.getDataLayout();
2674
2675 // A global is described by a structure
2676 // size_t beg;
2677 // size_t size;
2678 // size_t size_with_redzone;
2679 // const char *name;
2680 // const char *module_name;
2681 // size_t has_dynamic_init;
2682 // size_t padding_for_windows_msvc_incremental_link;
2683 // size_t odr_indicator;
2684 // We initialize an array of such structures and pass it to a run-time call.
2685 StructType *GlobalStructTy =
2686 StructType::get(elt1: IntptrTy, elts: IntptrTy, elts: IntptrTy, elts: IntptrTy, elts: IntptrTy,
2687 elts: IntptrTy, elts: IntptrTy, elts: IntptrTy);
2688 SmallVector<GlobalVariable *, 16> NewGlobals(n);
2689 SmallVector<Constant *, 16> Initializers(n);
2690
2691 for (size_t i = 0; i < n; i++) {
2692 GlobalVariable *G = GlobalsToChange[i];
2693
2694 GlobalValue::SanitizerMetadata MD;
2695 if (G->hasSanitizerMetadata())
2696 MD = G->getSanitizerMetadata();
2697
2698 // The runtime library tries demangling symbol names in the descriptor but
2699 // functionality like __cxa_demangle may be unavailable (e.g.
2700 // -static-libstdc++). So we demangle the symbol names here.
2701 std::string NameForGlobal = G->getName().str();
2702 GlobalVariable *Name =
2703 createPrivateGlobalForString(M, Str: llvm::demangle(MangledName: NameForGlobal),
2704 /*AllowMerging*/ true, NamePrefix: genName(suffix: "global"));
2705
2706 Type *Ty = G->getValueType();
2707 const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
2708 const uint64_t RightRedzoneSize = getRedzoneSizeForGlobal(SizeInBytes);
2709 Type *RightRedZoneTy = ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: RightRedzoneSize);
2710
2711 StructType *NewTy = StructType::get(elt1: Ty, elts: RightRedZoneTy);
2712 Constant *NewInitializer = ConstantStruct::get(
2713 T: NewTy, Vs: G->getInitializer(), Vs: Constant::getNullValue(Ty: RightRedZoneTy));
2714
2715 // Create a new global variable with enough space for a redzone.
2716 GlobalValue::LinkageTypes Linkage = G->getLinkage();
2717 if (G->isConstant() && Linkage == GlobalValue::PrivateLinkage)
2718 Linkage = GlobalValue::InternalLinkage;
2719 GlobalVariable *NewGlobal = new GlobalVariable(
2720 M, NewTy, G->isConstant(), Linkage, NewInitializer, "", G,
2721 G->getThreadLocalMode(), G->getAddressSpace());
2722 NewGlobal->copyAttributesFrom(Src: G);
2723 NewGlobal->setComdat(G->getComdat());
2724 NewGlobal->setAlignment(Align(getMinRedzoneSizeForGlobal()));
2725 // Don't fold globals with redzones. ODR violation detector and redzone
2726 // poisoning implicitly creates a dependence on the global's address, so it
2727 // is no longer valid for it to be marked unnamed_addr.
2728 NewGlobal->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
2729
2730 // Move null-terminated C strings to "__asan_cstring" section on Darwin.
2731 if (TargetTriple.isOSBinFormatMachO() && !G->hasSection() &&
2732 G->isConstant()) {
2733 auto Seq = dyn_cast<ConstantDataSequential>(Val: G->getInitializer());
2734 if (Seq && Seq->isCString())
2735 NewGlobal->setSection("__TEXT,__asan_cstring,regular");
2736 }
2737
2738 // Transfer the debug info and type metadata. The payload starts at offset
2739 // zero so we can copy the metadata over as is.
2740 NewGlobal->copyMetadata(Src: G, Offset: 0);
2741
2742 G->replaceAllUsesWith(V: NewGlobal);
2743 NewGlobal->takeName(V: G);
2744 G->eraseFromParent();
2745 NewGlobals[i] = NewGlobal;
2746
2747 Constant *ODRIndicator = Constant::getNullValue(Ty: IntptrTy);
2748 GlobalValue *InstrumentedGlobal = NewGlobal;
2749
2750 bool CanUsePrivateAliases =
2751 TargetTriple.isOSBinFormatELF() || TargetTriple.isOSBinFormatMachO() ||
2752 TargetTriple.isOSBinFormatWasm();
2753 if (CanUsePrivateAliases && UsePrivateAlias) {
2754 // Create local alias for NewGlobal to avoid crash on ODR between
2755 // instrumented and non-instrumented libraries.
2756 InstrumentedGlobal =
2757 GlobalAlias::create(Linkage: GlobalValue::PrivateLinkage, Name: "", Aliasee: NewGlobal);
2758 }
2759
2760 // ODR should not happen for local linkage.
2761 if (NewGlobal->hasLocalLinkage()) {
2762 ODRIndicator = ConstantInt::getAllOnesValue(Ty: IntptrTy);
2763 } else if (UseOdrIndicator) {
2764 // With local aliases, we need to provide another externally visible
2765 // symbol __odr_asan_XXX to detect ODR violation.
2766 auto *ODRIndicatorSym =
2767 new GlobalVariable(M, IRB.getInt8Ty(), false, Linkage,
2768 Constant::getNullValue(Ty: IRB.getInt8Ty()),
2769 kODRGenPrefix + NameForGlobal, nullptr,
2770 NewGlobal->getThreadLocalMode());
2771
2772 // Set meaningful attributes for indicator symbol.
2773 ODRIndicatorSym->setVisibility(NewGlobal->getVisibility());
2774 ODRIndicatorSym->setDLLStorageClass(NewGlobal->getDLLStorageClass());
2775 ODRIndicatorSym->setAlignment(Align(1));
2776 ODRIndicator = ConstantExpr::getPtrToInt(C: ODRIndicatorSym, Ty: IntptrTy);
2777 }
2778
2779 Constant *Initializer = ConstantStruct::get(
2780 T: GlobalStructTy,
2781 Vs: ConstantExpr::getPointerCast(C: InstrumentedGlobal, Ty: IntptrTy),
2782 Vs: ConstantInt::get(Ty: IntptrTy, V: SizeInBytes),
2783 Vs: ConstantInt::get(Ty: IntptrTy, V: SizeInBytes + RightRedzoneSize),
2784 Vs: ConstantExpr::getPointerCast(C: Name, Ty: IntptrTy),
2785 Vs: ConstantExpr::getPointerCast(C: getOrCreateModuleName(), Ty: IntptrTy),
2786 Vs: ConstantInt::get(Ty: IntptrTy, V: MD.IsDynInit),
2787 Vs: Constant::getNullValue(Ty: IntptrTy), Vs: ODRIndicator);
2788
2789 LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
2790
2791 Initializers[i] = Initializer;
2792 }
2793
2794 // Add instrumented globals to llvm.compiler.used list to avoid LTO from
2795 // ConstantMerge'ing them.
2796 SmallVector<GlobalValue *, 16> GlobalsToAddToUsedList;
2797 for (size_t i = 0; i < n; i++) {
2798 GlobalVariable *G = NewGlobals[i];
2799 if (G->getName().empty()) continue;
2800 GlobalsToAddToUsedList.push_back(Elt: G);
2801 }
2802 appendToCompilerUsed(M, Values: ArrayRef<GlobalValue *>(GlobalsToAddToUsedList));
2803
2804 if (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) {
2805 // Use COMDAT and register globals even if n == 0 to ensure that (a) the
2806 // linkage unit will only have one module constructor, and (b) the register
2807 // function will be called. The module destructor is not created when n ==
2808 // 0.
2809 *CtorComdat = true;
2810 instrumentGlobalsELF(IRB, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers, UniqueModuleId: getUniqueModuleId(M: &M));
2811 } else if (n == 0) {
2812 // When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
2813 // all compile units will have identical module constructor/destructor.
2814 *CtorComdat = TargetTriple.isOSBinFormatELF();
2815 } else {
2816 *CtorComdat = false;
2817 if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
2818 InstrumentGlobalsCOFF(IRB, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2819 } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
2820 InstrumentGlobalsMachO(IRB, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2821 } else {
2822 InstrumentGlobalsWithMetadataArray(IRB, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2823 }
2824 }
2825
2826 // Create calls for poisoning before initializers run and unpoisoning after.
2827 if (ClInitializers)
2828 createInitializerPoisonCalls();
2829
2830 LLVM_DEBUG(dbgs() << M);
2831}
2832
2833uint64_t
2834ModuleAddressSanitizer::getRedzoneSizeForGlobal(uint64_t SizeInBytes) const {
2835 constexpr uint64_t kMaxRZ = 1 << 18;
2836 const uint64_t MinRZ = getMinRedzoneSizeForGlobal();
2837
2838 uint64_t RZ = 0;
2839 if (SizeInBytes <= MinRZ / 2) {
2840 // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
2841 // at least 32 bytes, optimize when SizeInBytes is less than or equal to
2842 // half of MinRZ.
2843 RZ = MinRZ - SizeInBytes;
2844 } else {
2845 // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
2846 RZ = std::clamp(val: (SizeInBytes / MinRZ / 4) * MinRZ, lo: MinRZ, hi: kMaxRZ);
2847
2848 // Round up to multiple of MinRZ.
2849 if (SizeInBytes % MinRZ)
2850 RZ += MinRZ - (SizeInBytes % MinRZ);
2851 }
2852
2853 assert((RZ + SizeInBytes) % MinRZ == 0);
2854
2855 return RZ;
2856}
2857
2858int ModuleAddressSanitizer::GetAsanVersion() const {
2859 int LongSize = M.getDataLayout().getPointerSizeInBits();
2860 bool isAndroid = M.getTargetTriple().isAndroid();
2861 int Version = 8;
2862 // 32-bit Android is one version ahead because of the switch to dynamic
2863 // shadow.
2864 Version += (LongSize == 32 && isAndroid);
2865 return Version;
2866}
2867
2868GlobalVariable *ModuleAddressSanitizer::getOrCreateModuleName() {
2869 if (!ModuleName) {
2870 // We shouldn't merge same module names, as this string serves as unique
2871 // module ID in runtime.
2872 ModuleName =
2873 createPrivateGlobalForString(M, Str: M.getModuleIdentifier(),
2874 /*AllowMerging*/ false, NamePrefix: genName(suffix: "module"));
2875 }
2876 return ModuleName;
2877}
2878
2879bool ModuleAddressSanitizer::instrumentModule() {
2880 initializeCallbacks();
2881
2882 for (Function &F : M)
2883 removeASanIncompatibleFnAttributes(F, /*ReadsArgMem=*/false);
2884
2885 // Create a module constructor. A destructor is created lazily because not all
2886 // platforms, and not all modules need it.
2887 if (ConstructorKind == AsanCtorKind::Global) {
2888 if (CompileKernel) {
2889 // The kernel always builds with its own runtime, and therefore does not
2890 // need the init and version check calls.
2891 AsanCtorFunction = createSanitizerCtor(M, CtorName: kAsanModuleCtorName);
2892 } else {
2893 std::string AsanVersion = std::to_string(val: GetAsanVersion());
2894 std::string VersionCheckName =
2895 InsertVersionCheck ? (kAsanVersionCheckNamePrefix + AsanVersion) : "";
2896 std::tie(args&: AsanCtorFunction, args: std::ignore) =
2897 createSanitizerCtorAndInitFunctions(
2898 M, CtorName: kAsanModuleCtorName, InitName: kAsanInitName, /*InitArgTypes=*/{},
2899 /*InitArgs=*/{}, VersionCheckName);
2900 }
2901 }
2902
2903 bool CtorComdat = true;
2904 if (ClGlobals) {
2905 assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
2906 if (AsanCtorFunction) {
2907 IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
2908 instrumentGlobals(IRB, CtorComdat: &CtorComdat);
2909 } else {
2910 IRBuilder<> IRB(M);
2911 instrumentGlobals(IRB, CtorComdat: &CtorComdat);
2912 }
2913 }
2914
2915 const uint64_t Priority = GetCtorAndDtorPriority(TargetTriple);
2916
2917 // Put the constructor and destructor in comdat if both
2918 // (1) global instrumentation is not TU-specific
2919 // (2) target is ELF.
2920 if (UseCtorComdat && TargetTriple.isOSBinFormatELF() && CtorComdat) {
2921 if (AsanCtorFunction) {
2922 AsanCtorFunction->setComdat(M.getOrInsertComdat(Name: kAsanModuleCtorName));
2923 appendToGlobalCtors(M, F: AsanCtorFunction, Priority, Data: AsanCtorFunction);
2924 }
2925 if (AsanDtorFunction) {
2926 AsanDtorFunction->setComdat(M.getOrInsertComdat(Name: kAsanModuleDtorName));
2927 appendToGlobalDtors(M, F: AsanDtorFunction, Priority, Data: AsanDtorFunction);
2928 }
2929 } else {
2930 if (AsanCtorFunction)
2931 appendToGlobalCtors(M, F: AsanCtorFunction, Priority);
2932 if (AsanDtorFunction)
2933 appendToGlobalDtors(M, F: AsanDtorFunction, Priority);
2934 }
2935
2936 return true;
2937}
2938
2939void AddressSanitizer::initializeCallbacks(const TargetLibraryInfo *TLI) {
2940 IRBuilder<> IRB(M);
2941 // Create __asan_report* callbacks.
2942 // IsWrite, TypeSize and Exp are encoded in the function name.
2943 for (int Exp = 0; Exp < 2; Exp++) {
2944 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
2945 const std::string TypeStr = AccessIsWrite ? "store" : "load";
2946 const std::string ExpStr = Exp ? "exp_" : "";
2947 const std::string EndingStr = Recover ? "_noabort" : "";
2948
2949 SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
2950 SmallVector<Type *, 2> Args1{1, IntptrTy};
2951 AttributeList AL2;
2952 AttributeList AL1;
2953 if (Exp) {
2954 Type *ExpType = Type::getInt32Ty(C&: *C);
2955 Args2.push_back(Elt: ExpType);
2956 Args1.push_back(Elt: ExpType);
2957 if (auto AK = TLI->getExtAttrForI32Param(Signed: false)) {
2958 AL2 = AL2.addParamAttribute(C&: *C, ArgNo: 2, Kind: AK);
2959 AL1 = AL1.addParamAttribute(C&: *C, ArgNo: 1, Kind: AK);
2960 }
2961 }
2962 AsanErrorCallbackSized[AccessIsWrite][Exp] = Inserter.insertFunction(
2963 Name: kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
2964 Args: FunctionType::get(Result: IRB.getVoidTy(), Params: Args2, isVarArg: false), Args&: AL2);
2965
2966 AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] =
2967 Inserter.insertFunction(
2968 Name: ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
2969 Args: FunctionType::get(Result: IRB.getVoidTy(), Params: Args2, isVarArg: false), Args&: AL2);
2970
2971 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
2972 AccessSizeIndex++) {
2973 const std::string Suffix = TypeStr + itostr(X: 1ULL << AccessSizeIndex);
2974 AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] =
2975 Inserter.insertFunction(
2976 Name: kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr,
2977 Args: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false), Args&: AL1);
2978
2979 AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] =
2980 Inserter.insertFunction(
2981 Name: ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr,
2982 Args: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false), Args&: AL1);
2983 }
2984 }
2985 }
2986
2987 const std::string MemIntrinCallbackPrefix =
2988 (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
2989 ? std::string("")
2990 : ClMemoryAccessCallbackPrefix;
2991 AsanMemmove = Inserter.insertFunction(Name: MemIntrinCallbackPrefix + "memmove",
2992 Args&: PtrTy, Args&: PtrTy, Args&: PtrTy, Args&: IntptrTy);
2993 AsanMemcpy = Inserter.insertFunction(Name: MemIntrinCallbackPrefix + "memcpy",
2994 Args&: PtrTy, Args&: PtrTy, Args&: PtrTy, Args&: IntptrTy);
2995 AsanMemset =
2996 Inserter.insertFunction(Name: MemIntrinCallbackPrefix + "memset",
2997 Args: TLI->getAttrList(C, ArgNos: {1},
2998 /*Signed=*/false),
2999 Args&: PtrTy, Args&: PtrTy, Args: IRB.getInt32Ty(), Args&: IntptrTy);
3000
3001 AsanHandleNoReturnFunc =
3002 Inserter.insertFunction(Name: kAsanHandleNoReturnName, Args: IRB.getVoidTy());
3003
3004 AsanPtrCmpFunction =
3005 Inserter.insertFunction(Name: kAsanPtrCmp, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3006 AsanPtrSubFunction =
3007 Inserter.insertFunction(Name: kAsanPtrSub, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3008 if (Mapping.InGlobal)
3009 AsanShadowGlobal = M.getOrInsertGlobal(Name: "__asan_shadow",
3010 Ty: ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: 0));
3011
3012 AMDGPUAddressShared =
3013 Inserter.insertFunction(Name: kAMDGPUAddressSharedName, Args: IRB.getInt1Ty(), Args&: PtrTy);
3014 AMDGPUAddressPrivate = Inserter.insertFunction(Name: kAMDGPUAddressPrivateName,
3015 Args: IRB.getInt1Ty(), Args&: PtrTy);
3016}
3017
3018bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
3019 // For each NSObject descendant having a +load method, this method is invoked
3020 // by the ObjC runtime before any of the static constructors is called.
3021 // Therefore we need to instrument such methods with a call to __asan_init
3022 // at the beginning in order to initialize our runtime before any access to
3023 // the shadow memory.
3024 // We cannot just ignore these methods, because they may call other
3025 // instrumented functions.
3026 if (F.getName().contains(Other: " load]")) {
3027 FunctionCallee AsanInitFunction =
3028 declareSanitizerInitFunction(M&: *F.getParent(), InitName: kAsanInitName, InitArgTypes: {});
3029 IRBuilder<> IRB(F.front().begin());
3030 IRB.CreateCall(Callee: AsanInitFunction, Args: {});
3031 return true;
3032 }
3033 return false;
3034}
3035
3036bool AddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
3037 // Generate code only when dynamic addressing is needed.
3038 if (Mapping.Offset != kDynamicShadowSentinel)
3039 return false;
3040
3041 IRBuilder<> IRB(&F.front().front());
3042 if (Mapping.InGlobal) {
3043 if (ClWithIfuncSuppressRemat) {
3044 // An empty inline asm with input reg == output reg.
3045 // An opaque pointer-to-int cast, basically.
3046 InlineAsm *Asm = InlineAsm::get(
3047 Ty: FunctionType::get(Result: IntptrTy, Params: {AsanShadowGlobal->getType()}, isVarArg: false),
3048 AsmString: StringRef(""), Constraints: StringRef("=r,0"),
3049 /*hasSideEffects=*/false);
3050 LocalDynamicShadow =
3051 IRB.CreateCall(Callee: Asm, Args: {AsanShadowGlobal}, Name: ".asan.shadow");
3052 } else {
3053 LocalDynamicShadow =
3054 IRB.CreatePointerCast(V: AsanShadowGlobal, DestTy: IntptrTy, Name: ".asan.shadow");
3055 }
3056 } else {
3057 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
3058 Name: kAsanShadowMemoryDynamicAddress, Ty: IntptrTy);
3059 LocalDynamicShadow = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
3060 }
3061 return true;
3062}
3063
3064void AddressSanitizer::markEscapedLocalAllocas(Function &F) {
3065 // Find the one possible call to llvm.localescape and pre-mark allocas passed
3066 // to it as uninteresting. This assumes we haven't started processing allocas
3067 // yet. This check is done up front because iterating the use list in
3068 // isInterestingAlloca would be algorithmically slower.
3069 assert(ProcessedAllocas.empty() && "must process localescape before allocas");
3070
3071 // Try to get the declaration of llvm.localescape. If it's not in the module,
3072 // we can exit early.
3073 if (!F.getParent()->getFunction(Name: "llvm.localescape")) return;
3074
3075 // Look for a call to llvm.localescape call in the entry block. It can't be in
3076 // any other block.
3077 for (Instruction &I : F.getEntryBlock()) {
3078 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I);
3079 if (II && II->getIntrinsicID() == Intrinsic::localescape) {
3080 // We found a call. Mark all the allocas passed in as uninteresting.
3081 for (Value *Arg : II->args()) {
3082 AllocaInst *AI = dyn_cast<AllocaInst>(Val: Arg->stripPointerCasts());
3083 assert(AI && AI->isStaticAlloca() &&
3084 "non-static alloca arg to localescape");
3085 ProcessedAllocas[AI] = false;
3086 }
3087 break;
3088 }
3089 }
3090}
3091// Mitigation for https://github.com/google/sanitizers/issues/749
3092// We don't instrument Windows catch-block parameters to avoid
3093// interfering with exception handling assumptions.
3094void AddressSanitizer::markCatchParametersAsUninteresting(Function &F) {
3095 for (BasicBlock &BB : F) {
3096 for (Instruction &I : BB) {
3097 if (auto *CatchPad = dyn_cast<CatchPadInst>(Val: &I)) {
3098 // Mark the parameters to a catch-block as uninteresting to avoid
3099 // instrumenting them.
3100 for (Value *Operand : CatchPad->arg_operands())
3101 if (auto *AI = dyn_cast<AllocaInst>(Val: Operand))
3102 ProcessedAllocas[AI] = false;
3103 }
3104 }
3105 }
3106}
3107
3108bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
3109 bool ShouldInstrument =
3110 ClDebugMin < 0 || ClDebugMax < 0 ||
3111 (Instrumented >= ClDebugMin && Instrumented <= ClDebugMax);
3112 Instrumented++;
3113 return !ShouldInstrument;
3114}
3115
3116bool AddressSanitizer::instrumentFunction(Function &F,
3117 const TargetLibraryInfo *TLI,
3118 const TargetTransformInfo *TTI) {
3119 bool FunctionModified = false;
3120
3121 // Do not apply any instrumentation for naked functions.
3122 if (F.hasFnAttribute(Kind: Attribute::Naked))
3123 return FunctionModified;
3124
3125 // If needed, insert __asan_init before checking for SanitizeAddress attr.
3126 // This function needs to be called even if the function body is not
3127 // instrumented.
3128 if (maybeInsertAsanInitAtFunctionEntry(F))
3129 FunctionModified = true;
3130
3131 // Leave if the function doesn't need instrumentation.
3132 if (!F.hasFnAttribute(Kind: Attribute::SanitizeAddress)) return FunctionModified;
3133
3134 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
3135 return FunctionModified;
3136
3137 LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
3138
3139 initializeCallbacks(TLI);
3140
3141 FunctionStateRAII CleanupObj(this);
3142
3143 RuntimeCallInserter RTCI(F);
3144
3145 FunctionModified |= maybeInsertDynamicShadowAtFunctionEntry(F);
3146
3147 // We can't instrument allocas used with llvm.localescape. Only static allocas
3148 // can be passed to that intrinsic.
3149 markEscapedLocalAllocas(F);
3150
3151 if (TargetTriple.isOSWindows())
3152 markCatchParametersAsUninteresting(F);
3153
3154 // We want to instrument every address only once per basic block (unless there
3155 // are calls between uses).
3156 SmallPtrSet<Value *, 16> TempsToInstrument;
3157 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
3158 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
3159 SmallVector<Instruction *, 8> NoReturnCalls;
3160 SmallVector<BasicBlock *, 16> AllBlocks;
3161 SmallVector<Instruction *, 16> PointerComparisonsOrSubtracts;
3162
3163 // Fill the set of memory operations to instrument.
3164 for (auto &BB : F) {
3165 AllBlocks.push_back(Elt: &BB);
3166 TempsToInstrument.clear();
3167 int NumInsnsPerBB = 0;
3168 for (auto &Inst : BB) {
3169 if (LooksLikeCodeInBug11395(I: &Inst)) return false;
3170 // Skip instructions inserted by another instrumentation.
3171 if (Inst.hasMetadata(KindID: LLVMContext::MD_nosanitize))
3172 continue;
3173 SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
3174 getInterestingMemoryOperands(I: &Inst, Interesting&: InterestingOperands, TTI);
3175
3176 if (!InterestingOperands.empty()) {
3177 for (auto &Operand : InterestingOperands) {
3178 if (ClOpt && ClOptSameTemp) {
3179 Value *Ptr = Operand.getPtr();
3180 // If we have a mask, skip instrumentation if we've already
3181 // instrumented the full object. But don't add to TempsToInstrument
3182 // because we might get another load/store with a different mask.
3183 if (Operand.MaybeMask) {
3184 if (TempsToInstrument.count(Ptr))
3185 continue; // We've seen this (whole) temp in the current BB.
3186 } else {
3187 if (!TempsToInstrument.insert(Ptr).second)
3188 continue; // We've seen this temp in the current BB.
3189 }
3190 }
3191 OperandsToInstrument.push_back(Elt: Operand);
3192 NumInsnsPerBB++;
3193 }
3194 } else if (((ClInvalidPointerPairs || ClInvalidPointerCmp) &&
3195 isInterestingPointerComparison(I: &Inst)) ||
3196 ((ClInvalidPointerPairs || ClInvalidPointerSub) &&
3197 isInterestingPointerSubtraction(I: &Inst))) {
3198 PointerComparisonsOrSubtracts.push_back(Elt: &Inst);
3199 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
3200 // ok, take it.
3201 IntrinToInstrument.push_back(Elt: MI);
3202 NumInsnsPerBB++;
3203 } else {
3204 if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
3205 // A call inside BB.
3206 TempsToInstrument.clear();
3207 if (CB->doesNotReturn())
3208 NoReturnCalls.push_back(Elt: CB);
3209 }
3210 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
3211 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
3212 }
3213 if (NumInsnsPerBB >= ClMaxInsnsToInstrumentPerBB) break;
3214 }
3215 }
3216
3217 bool UseCalls = (InstrumentationWithCallsThreshold >= 0 &&
3218 OperandsToInstrument.size() + IntrinToInstrument.size() >
3219 (unsigned)InstrumentationWithCallsThreshold);
3220 const DataLayout &DL = F.getDataLayout();
3221 ObjectSizeOffsetVisitor ObjSizeVis(DL, TLI, F.getContext());
3222
3223 // Instrument.
3224 int NumInstrumented = 0;
3225 for (auto &Operand : OperandsToInstrument) {
3226 if (!suppressInstrumentationSiteForDebug(Instrumented&: NumInstrumented))
3227 instrumentMop(ObjSizeVis, O&: Operand, UseCalls,
3228 DL: F.getDataLayout(), RTCI);
3229 FunctionModified = true;
3230 }
3231 for (auto *Inst : IntrinToInstrument) {
3232 if (!suppressInstrumentationSiteForDebug(Instrumented&: NumInstrumented))
3233 instrumentMemIntrinsic(MI: Inst, RTCI);
3234 FunctionModified = true;
3235 }
3236
3237 FunctionStackPoisoner FSP(F, *this, RTCI);
3238 bool ChangedStack = FSP.runOnFunction();
3239
3240 // We must unpoison the stack before NoReturn calls (throw, _exit, etc).
3241 // See e.g. https://github.com/google/sanitizers/issues/37
3242 for (auto *CI : NoReturnCalls) {
3243 IRBuilder<> IRB(CI);
3244 RTCI.createRuntimeCall(IRB, Callee: AsanHandleNoReturnFunc, Args: {});
3245 }
3246
3247 for (auto *Inst : PointerComparisonsOrSubtracts) {
3248 FunctionModified |= instrumentPointerComparisonOrSubtraction(I: Inst, RTCI);
3249 }
3250
3251 if (ChangedStack || !NoReturnCalls.empty())
3252 FunctionModified = true;
3253
3254 LLVM_DEBUG(dbgs() << "ASAN done instrumenting: " << FunctionModified << " "
3255 << F << "\n");
3256
3257 return FunctionModified;
3258}
3259
3260// Workaround for bug 11395: we don't want to instrument stack in functions
3261// with large assembly blobs (32-bit only), otherwise reg alloc may crash.
3262// FIXME: remove once the bug 11395 is fixed.
3263bool AddressSanitizer::LooksLikeCodeInBug11395(Instruction *I) {
3264 if (LongSize != 32) return false;
3265 CallInst *CI = dyn_cast<CallInst>(Val: I);
3266 if (!CI || !CI->isInlineAsm()) return false;
3267 if (CI->arg_size() <= 5)
3268 return false;
3269 // We have inline assembly with quite a few arguments.
3270 return true;
3271}
3272
3273void FunctionStackPoisoner::initializeCallbacks(Module &M) {
3274 IRBuilder<> IRB(M);
3275 if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always ||
3276 ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
3277 const char *MallocNameTemplate =
3278 ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always
3279 ? kAsanStackMallocAlwaysNameTemplate
3280 : kAsanStackMallocNameTemplate;
3281 for (int Index = 0; Index <= kMaxAsanStackMallocSizeClass; Index++) {
3282 std::string Suffix = itostr(X: Index);
3283 AsanStackMallocFunc[Index] = ASan.Inserter.insertFunction(
3284 Name: MallocNameTemplate + Suffix, Args&: IntptrTy, Args&: IntptrTy);
3285 AsanStackFreeFunc[Index] =
3286 ASan.Inserter.insertFunction(Name: kAsanStackFreeNameTemplate + Suffix,
3287 Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3288 }
3289 }
3290 if (ASan.UseAfterScope) {
3291 AsanPoisonStackMemoryFunc = ASan.Inserter.insertFunction(
3292 Name: kAsanPoisonStackMemoryName, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3293 AsanUnpoisonStackMemoryFunc = ASan.Inserter.insertFunction(
3294 Name: kAsanUnpoisonStackMemoryName, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3295 }
3296
3297 for (size_t Val : {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xf1, 0xf2,
3298 0xf3, 0xf5, 0xf8}) {
3299 std::ostringstream Name;
3300 Name << kAsanSetShadowPrefix;
3301 Name << std::setw(2) << std::setfill('0') << std::hex << Val;
3302 AsanSetShadowFunc[Val] = ASan.Inserter.insertFunction(
3303 Name: Name.str(), Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3304 }
3305
3306 AsanAllocaPoisonFunc = ASan.Inserter.insertFunction(
3307 Name: kAsanAllocaPoison, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3308 AsanAllocasUnpoisonFunc = ASan.Inserter.insertFunction(
3309 Name: kAsanAllocasUnpoison, Args: IRB.getVoidTy(), Args&: IntptrTy, Args&: IntptrTy);
3310}
3311
3312void FunctionStackPoisoner::copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
3313 ArrayRef<uint8_t> ShadowBytes,
3314 size_t Begin, size_t End,
3315 IRBuilder<> &IRB,
3316 Value *ShadowBase) {
3317 if (Begin >= End)
3318 return;
3319
3320 const size_t LargestStoreSizeInBytes =
3321 std::min<size_t>(a: sizeof(uint64_t), b: ASan.LongSize / 8);
3322
3323 const bool IsLittleEndian = F.getDataLayout().isLittleEndian();
3324
3325 // Poison given range in shadow using larges store size with out leading and
3326 // trailing zeros in ShadowMask. Zeros never change, so they need neither
3327 // poisoning nor up-poisoning. Still we don't mind if some of them get into a
3328 // middle of a store.
3329 for (size_t i = Begin; i < End;) {
3330 if (!ShadowMask[i]) {
3331 assert(!ShadowBytes[i]);
3332 ++i;
3333 continue;
3334 }
3335
3336 size_t StoreSizeInBytes = LargestStoreSizeInBytes;
3337 // Fit store size into the range.
3338 while (StoreSizeInBytes > End - i)
3339 StoreSizeInBytes /= 2;
3340
3341 // Minimize store size by trimming trailing zeros.
3342 for (size_t j = StoreSizeInBytes - 1; j && !ShadowMask[i + j]; --j) {
3343 while (j <= StoreSizeInBytes / 2)
3344 StoreSizeInBytes /= 2;
3345 }
3346
3347 uint64_t Val = 0;
3348 for (size_t j = 0; j < StoreSizeInBytes; j++) {
3349 if (IsLittleEndian)
3350 Val |= (uint64_t)ShadowBytes[i + j] << (8 * j);
3351 else
3352 Val = (Val << 8) | ShadowBytes[i + j];
3353 }
3354
3355 Value *Ptr = IRB.CreateAdd(LHS: ShadowBase, RHS: ConstantInt::get(Ty: IntptrTy, V: i));
3356 Value *Poison = IRB.getIntN(N: StoreSizeInBytes * 8, C: Val);
3357 IRB.CreateAlignedStore(
3358 Val: Poison, Ptr: IRB.CreateIntToPtr(V: Ptr, DestTy: PointerType::getUnqual(C&: Poison->getContext())),
3359 Align: Align(1));
3360
3361 i += StoreSizeInBytes;
3362 }
3363}
3364
3365void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
3366 ArrayRef<uint8_t> ShadowBytes,
3367 IRBuilder<> &IRB, Value *ShadowBase) {
3368 copyToShadow(ShadowMask, ShadowBytes, Begin: 0, End: ShadowMask.size(), IRB, ShadowBase);
3369}
3370
3371void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
3372 ArrayRef<uint8_t> ShadowBytes,
3373 size_t Begin, size_t End,
3374 IRBuilder<> &IRB, Value *ShadowBase) {
3375 assert(ShadowMask.size() == ShadowBytes.size());
3376 size_t Done = Begin;
3377 for (size_t i = Begin, j = Begin + 1; i < End; i = j++) {
3378 if (!ShadowMask[i]) {
3379 assert(!ShadowBytes[i]);
3380 continue;
3381 }
3382 uint8_t Val = ShadowBytes[i];
3383 if (!AsanSetShadowFunc[Val])
3384 continue;
3385
3386 // Skip same values.
3387 for (; j < End && ShadowMask[j] && Val == ShadowBytes[j]; ++j) {
3388 }
3389
3390 if (j - i >= ASan.MaxInlinePoisoningSize) {
3391 copyToShadowInline(ShadowMask, ShadowBytes, Begin: Done, End: i, IRB, ShadowBase);
3392 RTCI.createRuntimeCall(
3393 IRB, Callee: AsanSetShadowFunc[Val],
3394 Args: {IRB.CreateAdd(LHS: ShadowBase, RHS: ConstantInt::get(Ty: IntptrTy, V: i)),
3395 ConstantInt::get(Ty: IntptrTy, V: j - i)});
3396 Done = j;
3397 }
3398 }
3399
3400 copyToShadowInline(ShadowMask, ShadowBytes, Begin: Done, End, IRB, ShadowBase);
3401}
3402
3403// Fake stack allocator (asan_fake_stack.h) has 11 size classes
3404// for every power of 2 from kMinStackMallocSize to kMaxAsanStackMallocSizeClass
3405static int StackMallocSizeClass(uint64_t LocalStackSize) {
3406 assert(LocalStackSize <= kMaxStackMallocSize);
3407 uint64_t MaxSize = kMinStackMallocSize;
3408 for (int i = 0;; i++, MaxSize *= 2)
3409 if (LocalStackSize <= MaxSize) return i;
3410 llvm_unreachable("impossible LocalStackSize");
3411}
3412
3413void FunctionStackPoisoner::copyArgsPassedByValToAllocas() {
3414 Instruction *CopyInsertPoint = &F.front().front();
3415 if (CopyInsertPoint == ASan.LocalDynamicShadow) {
3416 // Insert after the dynamic shadow location is determined
3417 CopyInsertPoint = CopyInsertPoint->getNextNode();
3418 assert(CopyInsertPoint);
3419 }
3420 IRBuilder<> IRB(CopyInsertPoint);
3421 const DataLayout &DL = F.getDataLayout();
3422 for (Argument &Arg : F.args()) {
3423 if (Arg.hasByValAttr()) {
3424 Type *Ty = Arg.getParamByValType();
3425 const Align Alignment =
3426 DL.getValueOrABITypeAlignment(Alignment: Arg.getParamAlign(), Ty);
3427
3428 AllocaInst *AI = IRB.CreateAlloca(
3429 Ty, ArraySize: nullptr,
3430 Name: (Arg.hasName() ? Arg.getName() : "Arg" + Twine(Arg.getArgNo())) +
3431 ".byval");
3432 AI->setAlignment(Alignment);
3433 Arg.replaceAllUsesWith(V: AI);
3434
3435 uint64_t AllocSize = DL.getTypeAllocSize(Ty);
3436 IRB.CreateMemCpy(Dst: AI, DstAlign: Alignment, Src: &Arg, SrcAlign: Alignment, Size: AllocSize);
3437 }
3438 }
3439}
3440
3441PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond,
3442 Value *ValueIfTrue,
3443 Instruction *ThenTerm,
3444 Value *ValueIfFalse) {
3445 PHINode *PHI = IRB.CreatePHI(Ty: ValueIfTrue->getType(), NumReservedValues: 2);
3446 BasicBlock *CondBlock = cast<Instruction>(Val: Cond)->getParent();
3447 PHI->addIncoming(V: ValueIfFalse, BB: CondBlock);
3448 BasicBlock *ThenBlock = ThenTerm->getParent();
3449 PHI->addIncoming(V: ValueIfTrue, BB: ThenBlock);
3450 return PHI;
3451}
3452
3453Value *FunctionStackPoisoner::createAllocaForLayout(
3454 IRBuilder<> &IRB, const ASanStackFrameLayout &L, bool Dynamic) {
3455 AllocaInst *Alloca;
3456 if (Dynamic) {
3457 Alloca = IRB.CreateAlloca(Ty: IRB.getInt8Ty(),
3458 ArraySize: ConstantInt::get(Ty: IRB.getInt64Ty(), V: L.FrameSize),
3459 Name: "MyAlloca");
3460 } else {
3461 Alloca = IRB.CreateAlloca(Ty: ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: L.FrameSize),
3462 ArraySize: nullptr, Name: "MyAlloca");
3463 assert(Alloca->isStaticAlloca());
3464 }
3465 assert((ClRealignStack & (ClRealignStack - 1)) == 0);
3466 uint64_t FrameAlignment = std::max(a: L.FrameAlignment, b: uint64_t(ClRealignStack));
3467 Alloca->setAlignment(Align(FrameAlignment));
3468 return Alloca;
3469}
3470
3471void FunctionStackPoisoner::createDynamicAllocasInitStorage() {
3472 BasicBlock &FirstBB = *F.begin();
3473 IRBuilder<> IRB(dyn_cast<Instruction>(Val: FirstBB.begin()));
3474 DynamicAllocaLayout = IRB.CreateAlloca(Ty: IntptrTy, ArraySize: nullptr);
3475 IRB.CreateStore(Val: Constant::getNullValue(Ty: IntptrTy), Ptr: DynamicAllocaLayout);
3476 DynamicAllocaLayout->setAlignment(Align(32));
3477}
3478
3479void FunctionStackPoisoner::processDynamicAllocas() {
3480 if (!ClInstrumentDynamicAllocas || DynamicAllocaVec.empty()) {
3481 assert(DynamicAllocaPoisonCallVec.empty());
3482 return;
3483 }
3484
3485 // Insert poison calls for lifetime intrinsics for dynamic allocas.
3486 for (const auto &APC : DynamicAllocaPoisonCallVec) {
3487 assert(APC.InsBefore);
3488 assert(APC.AI);
3489 assert(ASan.isInterestingAlloca(*APC.AI));
3490 assert(!APC.AI->isStaticAlloca());
3491
3492 IRBuilder<> IRB(APC.InsBefore);
3493 poisonAlloca(V: APC.AI, Size: APC.Size, IRB, DoPoison: APC.DoPoison);
3494 // Dynamic allocas will be unpoisoned unconditionally below in
3495 // unpoisonDynamicAllocas.
3496 // Flag that we need unpoison static allocas.
3497 }
3498
3499 // Handle dynamic allocas.
3500 createDynamicAllocasInitStorage();
3501 for (auto &AI : DynamicAllocaVec)
3502 handleDynamicAllocaCall(AI);
3503 unpoisonDynamicAllocas();
3504}
3505
3506/// Collect instructions in the entry block after \p InsBefore which initialize
3507/// permanent storage for a function argument. These instructions must remain in
3508/// the entry block so that uninitialized values do not appear in backtraces. An
3509/// added benefit is that this conserves spill slots. This does not move stores
3510/// before instrumented / "interesting" allocas.
3511static void findStoresToUninstrumentedArgAllocas(
3512 AddressSanitizer &ASan, Instruction &InsBefore,
3513 SmallVectorImpl<Instruction *> &InitInsts) {
3514 Instruction *Start = InsBefore.getNextNode();
3515 for (Instruction *It = Start; It; It = It->getNextNode()) {
3516 // Argument initialization looks like:
3517 // 1) store <Argument>, <Alloca> OR
3518 // 2) <CastArgument> = cast <Argument> to ...
3519 // store <CastArgument> to <Alloca>
3520 // Do not consider any other kind of instruction.
3521 //
3522 // Note: This covers all known cases, but may not be exhaustive. An
3523 // alternative to pattern-matching stores is to DFS over all Argument uses:
3524 // this might be more general, but is probably much more complicated.
3525 if (isa<AllocaInst>(Val: It) || isa<CastInst>(Val: It))
3526 continue;
3527 if (auto *Store = dyn_cast<StoreInst>(Val: It)) {
3528 // The store destination must be an alloca that isn't interesting for
3529 // ASan to instrument. These are moved up before InsBefore, and they're
3530 // not interesting because allocas for arguments can be mem2reg'd.
3531 auto *Alloca = dyn_cast<AllocaInst>(Val: Store->getPointerOperand());
3532 if (!Alloca || ASan.isInterestingAlloca(AI: *Alloca))
3533 continue;
3534
3535 Value *Val = Store->getValueOperand();
3536 bool IsDirectArgInit = isa<Argument>(Val);
3537 bool IsArgInitViaCast =
3538 isa<CastInst>(Val) &&
3539 isa<Argument>(Val: cast<CastInst>(Val)->getOperand(i_nocapture: 0)) &&
3540 // Check that the cast appears directly before the store. Otherwise
3541 // moving the cast before InsBefore may break the IR.
3542 Val == It->getPrevNode();
3543 bool IsArgInit = IsDirectArgInit || IsArgInitViaCast;
3544 if (!IsArgInit)
3545 continue;
3546
3547 if (IsArgInitViaCast)
3548 InitInsts.push_back(Elt: cast<Instruction>(Val));
3549 InitInsts.push_back(Elt: Store);
3550 continue;
3551 }
3552
3553 // Do not reorder past unknown instructions: argument initialization should
3554 // only involve casts and stores.
3555 return;
3556 }
3557}
3558
3559static StringRef getAllocaName(AllocaInst *AI) {
3560 // Alloca could have been renamed for uniqueness. Its true name will have been
3561 // recorded as an annotation.
3562 if (AI->hasMetadata(KindID: LLVMContext::MD_annotation)) {
3563 MDTuple *AllocaAnnotations =
3564 cast<MDTuple>(Val: AI->getMetadata(KindID: LLVMContext::MD_annotation));
3565 for (auto &Annotation : AllocaAnnotations->operands()) {
3566 if (!isa<MDTuple>(Val: Annotation))
3567 continue;
3568 auto AnnotationTuple = cast<MDTuple>(Val: Annotation);
3569 for (unsigned Index = 0; Index < AnnotationTuple->getNumOperands();
3570 Index++) {
3571 // All annotations are strings
3572 auto MetadataString =
3573 cast<MDString>(Val: AnnotationTuple->getOperand(I: Index));
3574 if (MetadataString->getString() == "alloca_name_altered")
3575 return cast<MDString>(Val: AnnotationTuple->getOperand(I: Index + 1))
3576 ->getString();
3577 }
3578 }
3579 }
3580 return AI->getName();
3581}
3582
3583void FunctionStackPoisoner::processStaticAllocas() {
3584 if (AllocaVec.empty()) {
3585 assert(StaticAllocaPoisonCallVec.empty());
3586 return;
3587 }
3588
3589 int StackMallocIdx = -1;
3590 DebugLoc EntryDebugLocation;
3591 if (auto SP = F.getSubprogram())
3592 EntryDebugLocation =
3593 DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
3594
3595 Instruction *InsBefore = AllocaVec[0];
3596 IRBuilder<> IRB(InsBefore);
3597
3598 // Make sure non-instrumented allocas stay in the entry block. Otherwise,
3599 // debug info is broken, because only entry-block allocas are treated as
3600 // regular stack slots.
3601 auto InsBeforeB = InsBefore->getParent();
3602 assert(InsBeforeB == &F.getEntryBlock());
3603 for (auto *AI : StaticAllocasToMoveUp)
3604 if (AI->getParent() == InsBeforeB)
3605 AI->moveBefore(InsertPos: InsBefore->getIterator());
3606
3607 // Move stores of arguments into entry-block allocas as well. This prevents
3608 // extra stack slots from being generated (to house the argument values until
3609 // they can be stored into the allocas). This also prevents uninitialized
3610 // values from being shown in backtraces.
3611 SmallVector<Instruction *, 8> ArgInitInsts;
3612 findStoresToUninstrumentedArgAllocas(ASan, InsBefore&: *InsBefore, InitInsts&: ArgInitInsts);
3613 for (Instruction *ArgInitInst : ArgInitInsts)
3614 ArgInitInst->moveBefore(InsertPos: InsBefore->getIterator());
3615
3616 // If we have a call to llvm.localescape, keep it in the entry block.
3617 if (LocalEscapeCall)
3618 LocalEscapeCall->moveBefore(InsertPos: InsBefore->getIterator());
3619
3620 SmallVector<ASanStackVariableDescription, 16> SVD;
3621 SVD.reserve(N: AllocaVec.size());
3622 for (AllocaInst *AI : AllocaVec) {
3623 StringRef Name = getAllocaName(AI);
3624 ASanStackVariableDescription D = {.Name: Name.data(),
3625 .Size: ASan.getAllocaSizeInBytes(AI: *AI),
3626 .LifetimeSize: 0,
3627 .Alignment: AI->getAlign().value(),
3628 .AI: AI,
3629 .Offset: 0,
3630 .Line: 0};
3631 SVD.push_back(Elt: D);
3632 }
3633
3634 // Minimal header size (left redzone) is 4 pointers,
3635 // i.e. 32 bytes on 64-bit platforms and 16 bytes in 32-bit platforms.
3636 uint64_t Granularity = 1ULL << Mapping.Scale;
3637 uint64_t MinHeaderSize = std::max(a: (uint64_t)ASan.LongSize / 2, b: Granularity);
3638 const ASanStackFrameLayout &L =
3639 ComputeASanStackFrameLayout(Vars&: SVD, Granularity, MinHeaderSize);
3640
3641 // Build AllocaToSVDMap for ASanStackVariableDescription lookup.
3642 DenseMap<const AllocaInst *, ASanStackVariableDescription *> AllocaToSVDMap;
3643 for (auto &Desc : SVD)
3644 AllocaToSVDMap[Desc.AI] = &Desc;
3645
3646 // Update SVD with information from lifetime intrinsics.
3647 for (const auto &APC : StaticAllocaPoisonCallVec) {
3648 assert(APC.InsBefore);
3649 assert(APC.AI);
3650 assert(ASan.isInterestingAlloca(*APC.AI));
3651 assert(APC.AI->isStaticAlloca());
3652
3653 ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
3654 Desc.LifetimeSize = Desc.Size;
3655 if (const DILocation *FnLoc = EntryDebugLocation.get()) {
3656 if (const DILocation *LifetimeLoc = APC.InsBefore->getDebugLoc().get()) {
3657 if (LifetimeLoc->getFile() == FnLoc->getFile())
3658 if (unsigned Line = LifetimeLoc->getLine())
3659 Desc.Line = std::min(a: Desc.Line ? Desc.Line : Line, b: Line);
3660 }
3661 }
3662 }
3663
3664 auto DescriptionString = ComputeASanStackFrameDescription(Vars: SVD);
3665 LLVM_DEBUG(dbgs() << DescriptionString << " --- " << L.FrameSize << "\n");
3666 uint64_t LocalStackSize = L.FrameSize;
3667 bool DoStackMalloc =
3668 ASan.UseAfterReturn != AsanDetectStackUseAfterReturnMode::Never &&
3669 !ASan.CompileKernel && LocalStackSize <= kMaxStackMallocSize;
3670 bool DoDynamicAlloca = ClDynamicAllocaStack;
3671 // Don't do dynamic alloca or stack malloc if:
3672 // 1) There is inline asm: too often it makes assumptions on which registers
3673 // are available.
3674 // 2) There is a returns_twice call (typically setjmp), which is
3675 // optimization-hostile, and doesn't play well with introduced indirect
3676 // register-relative calculation of local variable addresses.
3677 DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall;
3678 DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall;
3679
3680 Type *PtrTy = F.getDataLayout().getAllocaPtrType(Ctx&: F.getContext());
3681 Value *StaticAlloca =
3682 DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, Dynamic: false);
3683
3684 Value *FakeStackPtr;
3685 Value *FakeStackInt;
3686 Value *LocalStackBase;
3687 Value *LocalStackBaseAlloca;
3688 uint8_t DIExprFlags = DIExpression::ApplyOffset;
3689
3690 if (DoStackMalloc) {
3691 LocalStackBaseAlloca =
3692 IRB.CreateAlloca(Ty: IntptrTy, ArraySize: nullptr, Name: "asan_local_stack_base");
3693 if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
3694 // void *FakeStack = __asan_option_detect_stack_use_after_return
3695 // ? __asan_stack_malloc_N(LocalStackSize)
3696 // : nullptr;
3697 // void *LocalStackBase = (FakeStack) ? FakeStack :
3698 // alloca(LocalStackSize);
3699 Constant *OptionDetectUseAfterReturn = F.getParent()->getOrInsertGlobal(
3700 Name: kAsanOptionDetectUseAfterReturn, Ty: IRB.getInt32Ty());
3701 Value *UseAfterReturnIsEnabled = IRB.CreateICmpNE(
3702 LHS: IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: OptionDetectUseAfterReturn),
3703 RHS: Constant::getNullValue(Ty: IRB.getInt32Ty()));
3704 Instruction *Term =
3705 SplitBlockAndInsertIfThen(Cond: UseAfterReturnIsEnabled, SplitBefore: InsBefore, Unreachable: false);
3706 IRBuilder<> IRBIf(Term);
3707 StackMallocIdx = StackMallocSizeClass(LocalStackSize);
3708 assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass);
3709 Value *FakeStackValue =
3710 RTCI.createRuntimeCall(IRB&: IRBIf, Callee: AsanStackMallocFunc[StackMallocIdx],
3711 Args: ConstantInt::get(Ty: IntptrTy, V: LocalStackSize));
3712 IRB.SetInsertPoint(InsBefore);
3713 FakeStackInt = createPHI(IRB, Cond: UseAfterReturnIsEnabled, ValueIfTrue: FakeStackValue,
3714 ThenTerm: Term, ValueIfFalse: ConstantInt::get(Ty: IntptrTy, V: 0));
3715 } else {
3716 // assert(ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode:Always)
3717 // void *FakeStack = __asan_stack_malloc_N(LocalStackSize);
3718 // void *LocalStackBase = (FakeStack) ? FakeStack :
3719 // alloca(LocalStackSize);
3720 StackMallocIdx = StackMallocSizeClass(LocalStackSize);
3721 FakeStackInt =
3722 RTCI.createRuntimeCall(IRB, Callee: AsanStackMallocFunc[StackMallocIdx],
3723 Args: ConstantInt::get(Ty: IntptrTy, V: LocalStackSize));
3724 }
3725 FakeStackPtr = IRB.CreateIntToPtr(V: FakeStackInt, DestTy: PtrTy);
3726 Value *NoFakeStack =
3727 IRB.CreateICmpEQ(LHS: FakeStackInt, RHS: Constant::getNullValue(Ty: IntptrTy));
3728 Instruction *Term =
3729 SplitBlockAndInsertIfThen(Cond: NoFakeStack, SplitBefore: InsBefore, Unreachable: false);
3730 IRBuilder<> IRBIf(Term);
3731 Value *AllocaValue =
3732 DoDynamicAlloca ? createAllocaForLayout(IRB&: IRBIf, L, Dynamic: true) : StaticAlloca;
3733
3734 IRB.SetInsertPoint(InsBefore);
3735 LocalStackBase =
3736 createPHI(IRB, Cond: NoFakeStack, ValueIfTrue: AllocaValue, ThenTerm: Term, ValueIfFalse: FakeStackPtr);
3737 IRB.CreateStore(Val: LocalStackBase, Ptr: LocalStackBaseAlloca);
3738 DIExprFlags |= DIExpression::DerefBefore;
3739 } else {
3740 // void *FakeStack = nullptr;
3741 // void *LocalStackBase = alloca(LocalStackSize);
3742 FakeStackInt = Constant::getNullValue(Ty: IntptrTy);
3743 FakeStackPtr = Constant::getNullValue(Ty: PtrTy);
3744 LocalStackBase =
3745 DoDynamicAlloca ? createAllocaForLayout(IRB, L, Dynamic: true) : StaticAlloca;
3746 LocalStackBaseAlloca = LocalStackBase;
3747 }
3748
3749 // Replace Alloca instructions with base+offset.
3750 SmallVector<Value *> NewAllocaPtrs;
3751 for (const auto &Desc : SVD) {
3752 AllocaInst *AI = Desc.AI;
3753 replaceDbgDeclare(Address: AI, NewAddress: LocalStackBaseAlloca, Builder&: DIB, DIExprFlags, Offset: Desc.Offset);
3754 Value *NewAllocaPtr = IRB.CreatePtrAdd(
3755 Ptr: LocalStackBase, Offset: ConstantInt::get(Ty: IntptrTy, V: Desc.Offset));
3756 if (NewAllocaPtr->getType() != AI->getType())
3757 NewAllocaPtr = IRB.CreateAddrSpaceCast(V: NewAllocaPtr, DestTy: AI->getType());
3758 AI->replaceAllUsesWith(V: NewAllocaPtr);
3759 NewAllocaPtrs.push_back(Elt: NewAllocaPtr);
3760 }
3761
3762 // The left-most redzone has enough space for at least 4 pointers.
3763 // Write the Magic value to redzone[0].
3764 IRB.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: kCurrentStackFrameMagic),
3765 Ptr: LocalStackBase);
3766 // Write the frame description constant to redzone[1].
3767 Value *BasePlus1 = IRB.CreatePtrAdd(
3768 Ptr: LocalStackBase, Offset: ConstantInt::get(Ty: IntptrTy, V: ASan.LongSize / 8));
3769 GlobalVariable *StackDescriptionGlobal =
3770 createPrivateGlobalForString(M&: *F.getParent(), Str: DescriptionString,
3771 /*AllowMerging*/ true, NamePrefix: genName(suffix: "stack"));
3772 Value *Description = IRB.CreatePointerCast(V: StackDescriptionGlobal, DestTy: IntptrTy);
3773 IRB.CreateStore(Val: Description, Ptr: BasePlus1);
3774 // Write the PC to redzone[2].
3775 Value *BasePlus2 = IRB.CreatePtrAdd(
3776 Ptr: LocalStackBase, Offset: ConstantInt::get(Ty: IntptrTy, V: 2 * ASan.LongSize / 8));
3777 IRB.CreateStore(Val: IRB.CreatePointerCast(V: &F, DestTy: IntptrTy), Ptr: BasePlus2);
3778
3779 const auto &ShadowAfterScope = GetShadowBytesAfterScope(Vars: SVD, Layout: L);
3780
3781 // Poison the stack red zones at the entry.
3782 Value *ShadowBase =
3783 ASan.memToShadow(Shadow: IRB.CreatePtrToInt(V: LocalStackBase, DestTy: IntptrTy), IRB);
3784 // As mask we must use most poisoned case: red zones and after scope.
3785 // As bytes we can use either the same or just red zones only.
3786 copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowAfterScope, IRB, ShadowBase);
3787
3788 if (!StaticAllocaPoisonCallVec.empty()) {
3789 const auto &ShadowInScope = GetShadowBytes(Vars: SVD, Layout: L);
3790
3791 // Poison static allocas near lifetime intrinsics.
3792 for (const auto &APC : StaticAllocaPoisonCallVec) {
3793 const ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
3794 assert(Desc.Offset % L.Granularity == 0);
3795 size_t Begin = Desc.Offset / L.Granularity;
3796 size_t End = Begin + (APC.Size + L.Granularity - 1) / L.Granularity;
3797
3798 IRBuilder<> IRB(APC.InsBefore);
3799 copyToShadow(ShadowMask: ShadowAfterScope,
3800 ShadowBytes: APC.DoPoison ? ShadowAfterScope : ShadowInScope, Begin, End,
3801 IRB, ShadowBase);
3802 }
3803 }
3804
3805 // Remove lifetime markers now that these are no longer allocas.
3806 for (Value *NewAllocaPtr : NewAllocaPtrs) {
3807 for (User *U : make_early_inc_range(Range: NewAllocaPtr->users())) {
3808 auto *I = cast<Instruction>(Val: U);
3809 if (I->isLifetimeStartOrEnd())
3810 I->eraseFromParent();
3811 }
3812 }
3813
3814 SmallVector<uint8_t, 64> ShadowClean(ShadowAfterScope.size(), 0);
3815 SmallVector<uint8_t, 64> ShadowAfterReturn;
3816
3817 // (Un)poison the stack before all ret instructions.
3818 for (Instruction *Ret : RetVec) {
3819 IRBuilder<> IRBRet(Ret);
3820 // Mark the current frame as retired.
3821 IRBRet.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: kRetiredStackFrameMagic),
3822 Ptr: LocalStackBase);
3823 if (DoStackMalloc) {
3824 assert(StackMallocIdx >= 0);
3825 // if FakeStack != 0 // LocalStackBase == FakeStack
3826 // // In use-after-return mode, poison the whole stack frame.
3827 // if StackMallocIdx <= 4
3828 // // For small sizes inline the whole thing:
3829 // memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
3830 // **SavedFlagPtr(FakeStack) = 0
3831 // else
3832 // __asan_stack_free_N(FakeStack, LocalStackSize)
3833 // else
3834 // <This is not a fake stack; unpoison the redzones>
3835 Value *Cmp =
3836 IRBRet.CreateICmpNE(LHS: FakeStackInt, RHS: Constant::getNullValue(Ty: IntptrTy));
3837 Instruction *ThenTerm, *ElseTerm;
3838 SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: Ret, ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
3839
3840 IRBuilder<> IRBPoison(ThenTerm);
3841 if (ASan.MaxInlinePoisoningSize != 0 && StackMallocIdx <= 4) {
3842 int ClassSize = kMinStackMallocSize << StackMallocIdx;
3843 ShadowAfterReturn.resize(N: ClassSize / L.Granularity,
3844 NV: kAsanStackUseAfterReturnMagic);
3845 copyToShadow(ShadowMask: ShadowAfterReturn, ShadowBytes: ShadowAfterReturn, IRB&: IRBPoison,
3846 ShadowBase);
3847 Value *SavedFlagPtrPtr = IRBPoison.CreatePtrAdd(
3848 Ptr: FakeStackPtr,
3849 Offset: ConstantInt::get(Ty: IntptrTy, V: ClassSize - ASan.LongSize / 8));
3850 Value *SavedFlagPtr = IRBPoison.CreateLoad(Ty: IntptrTy, Ptr: SavedFlagPtrPtr);
3851 IRBPoison.CreateStore(
3852 Val: Constant::getNullValue(Ty: IRBPoison.getInt8Ty()),
3853 Ptr: IRBPoison.CreateIntToPtr(V: SavedFlagPtr, DestTy: IRBPoison.getPtrTy()));
3854 } else {
3855 // For larger frames call __asan_stack_free_*.
3856 RTCI.createRuntimeCall(
3857 IRB&: IRBPoison, Callee: AsanStackFreeFunc[StackMallocIdx],
3858 Args: {FakeStackInt, ConstantInt::get(Ty: IntptrTy, V: LocalStackSize)});
3859 }
3860
3861 IRBuilder<> IRBElse(ElseTerm);
3862 copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowClean, IRB&: IRBElse, ShadowBase);
3863 } else {
3864 copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowClean, IRB&: IRBRet, ShadowBase);
3865 }
3866 }
3867
3868 // We are done. Remove the old unused alloca instructions.
3869 for (auto *AI : AllocaVec)
3870 AI->eraseFromParent();
3871}
3872
3873void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size,
3874 IRBuilder<> &IRB, bool DoPoison) {
3875 // For now just insert the call to ASan runtime.
3876 Value *AddrArg = IRB.CreatePointerCast(V, DestTy: IntptrTy);
3877 Value *SizeArg = ConstantInt::get(Ty: IntptrTy, V: Size);
3878 RTCI.createRuntimeCall(
3879 IRB, Callee: DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc,
3880 Args: {AddrArg, SizeArg});
3881}
3882
3883// Handling llvm.lifetime intrinsics for a given %alloca:
3884// (1) collect all llvm.lifetime.xxx(%size, %value) describing the alloca.
3885// (2) if %size is constant, poison memory for llvm.lifetime.end (to detect
3886// invalid accesses) and unpoison it for llvm.lifetime.start (the memory
3887// could be poisoned by previous llvm.lifetime.end instruction, as the
3888// variable may go in and out of scope several times, e.g. in loops).
3889// (3) if we poisoned at least one %alloca in a function,
3890// unpoison the whole stack frame at function exit.
3891void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) {
3892 IRBuilder<> IRB(AI);
3893
3894 const Align Alignment = std::max(a: Align(kAllocaRzSize), b: AI->getAlign());
3895 const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1;
3896
3897 Value *Zero = Constant::getNullValue(Ty: IntptrTy);
3898 Value *AllocaRzSize = ConstantInt::get(Ty: IntptrTy, V: kAllocaRzSize);
3899 Value *AllocaRzMask = ConstantInt::get(Ty: IntptrTy, V: AllocaRedzoneMask);
3900
3901 // Since we need to extend alloca with additional memory to locate
3902 // redzones, and OldSize is number of allocated blocks with
3903 // ElementSize size, get allocated memory size in bytes by
3904 // OldSize * ElementSize.
3905 Value *OldSize = IRB.CreateAllocationSize(DestTy: IntptrTy, AI);
3906
3907 // PartialSize = OldSize % 32
3908 Value *PartialSize = IRB.CreateAnd(LHS: OldSize, RHS: AllocaRzMask);
3909
3910 // Misalign = kAllocaRzSize - PartialSize;
3911 Value *Misalign = IRB.CreateSub(LHS: AllocaRzSize, RHS: PartialSize);
3912
3913 // PartialPadding = Misalign != kAllocaRzSize ? Misalign : 0;
3914 Value *Cond = IRB.CreateICmpNE(LHS: Misalign, RHS: AllocaRzSize);
3915 Value *PartialPadding = IRB.CreateSelect(C: Cond, True: Misalign, False: Zero);
3916
3917 // AdditionalChunkSize = Alignment + PartialPadding + kAllocaRzSize
3918 // Alignment is added to locate left redzone, PartialPadding for possible
3919 // partial redzone and kAllocaRzSize for right redzone respectively.
3920 Value *AdditionalChunkSize = IRB.CreateAdd(
3921 LHS: ConstantInt::get(Ty: IntptrTy, V: Alignment.value() + kAllocaRzSize),
3922 RHS: PartialPadding);
3923
3924 Value *NewSize = IRB.CreateAdd(LHS: OldSize, RHS: AdditionalChunkSize);
3925
3926 // Insert new alloca with new NewSize and Alignment params.
3927 AllocaInst *NewAlloca = IRB.CreateAlloca(Ty: IRB.getInt8Ty(), ArraySize: NewSize);
3928 NewAlloca->setAlignment(Alignment);
3929
3930 // NewAddress = Address + Alignment
3931 Value *NewAddress =
3932 IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: NewAlloca, DestTy: IntptrTy),
3933 RHS: ConstantInt::get(Ty: IntptrTy, V: Alignment.value()));
3934
3935 // Insert __asan_alloca_poison call for new created alloca.
3936 RTCI.createRuntimeCall(IRB, Callee: AsanAllocaPoisonFunc, Args: {NewAddress, OldSize});
3937
3938 // Store the last alloca's address to DynamicAllocaLayout. We'll need this
3939 // for unpoisoning stuff.
3940 IRB.CreateStore(Val: IRB.CreatePtrToInt(V: NewAlloca, DestTy: IntptrTy), Ptr: DynamicAllocaLayout);
3941
3942 Value *NewAddressPtr = IRB.CreateIntToPtr(V: NewAddress, DestTy: AI->getType());
3943
3944 // Remove lifetime markers now that this is no longer an alloca.
3945 for (User *U : make_early_inc_range(Range: AI->users())) {
3946 auto *I = cast<Instruction>(Val: U);
3947 if (I->isLifetimeStartOrEnd())
3948 I->eraseFromParent();
3949 }
3950
3951 // Replace all uses of AddressReturnedByAlloca with NewAddressPtr.
3952 AI->replaceAllUsesWith(V: NewAddressPtr);
3953
3954 // We are done. Erase old alloca from parent.
3955 AI->eraseFromParent();
3956}
3957
3958// isSafeAccess returns true if Addr is always inbounds with respect to its
3959// base object. For example, it is a field access or an array access with
3960// constant inbounds index.
3961bool AddressSanitizer::isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis,
3962 Value *Addr, TypeSize TypeStoreSize) const {
3963 if (TypeStoreSize.isScalable())
3964 // TODO: We can use vscale_range to convert a scalable value to an
3965 // upper bound on the access size.
3966 return false;
3967
3968 SizeOffsetAPInt SizeOffset = ObjSizeVis.compute(V: Addr);
3969 if (!SizeOffset.bothKnown())
3970 return false;
3971
3972 uint64_t Size = SizeOffset.Size.getZExtValue();
3973 int64_t Offset = SizeOffset.Offset.getSExtValue();
3974
3975 // Three checks are required to ensure safety:
3976 // . Offset >= 0 (since the offset is given from the base ptr)
3977 // . Size >= Offset (unsigned)
3978 // . Size - Offset >= NeededSize (unsigned)
3979 return Offset >= 0 && Size >= uint64_t(Offset) &&
3980 Size - uint64_t(Offset) >= TypeStoreSize / 8;
3981}
3982