1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html)
15/// We associate a few shadow bits with every byte of the application memory,
16/// poison the shadow of the malloc-ed or alloca-ed memory, load the shadow,
17/// bits on every memory read, propagate the shadow bits through some of the
18/// arithmetic instruction (including MOV), store the shadow bits on every
19/// memory write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// Note that the sanitizer code has to deal with how shadow/origin pairs
126/// returned by the these functions are represented in different ABIs. In
127/// the X86_64 ABI they are returned in RDX:RAX, in PowerPC64 they are
128/// returned in r3 and r4, and in the SystemZ ABI they are written to memory
129/// pointed to by a hidden parameter.
130/// - TLS variables are stored in a single per-task struct. A call to a
131/// function __msan_get_context_state() returning a pointer to that struct
132/// is inserted into every instrumented function before the entry block;
133/// - __msan_warning() takes a 32-bit origin parameter;
134/// - local variables are poisoned with __msan_poison_alloca() upon function
135/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
136/// function;
137/// - the pass doesn't declare any global variables or add global constructors
138/// to the translation unit.
139///
140/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
141/// calls, making sure we're on the safe side wrt. possible false positives.
142///
143/// KernelMemorySanitizer only supports X86_64, SystemZ and PowerPC64 at the
144/// moment.
145///
146//
147// FIXME: This sanitizer does not yet handle scalable vectors
148//
149//===----------------------------------------------------------------------===//
150
151#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
152#include "llvm/ADT/APInt.h"
153#include "llvm/ADT/ArrayRef.h"
154#include "llvm/ADT/DenseMap.h"
155#include "llvm/ADT/DepthFirstIterator.h"
156#include "llvm/ADT/SetVector.h"
157#include "llvm/ADT/SmallPtrSet.h"
158#include "llvm/ADT/SmallVector.h"
159#include "llvm/ADT/StringExtras.h"
160#include "llvm/ADT/StringRef.h"
161#include "llvm/Analysis/GlobalsModRef.h"
162#include "llvm/Analysis/TargetLibraryInfo.h"
163#include "llvm/Analysis/ValueTracking.h"
164#include "llvm/IR/Argument.h"
165#include "llvm/IR/AttributeMask.h"
166#include "llvm/IR/Attributes.h"
167#include "llvm/IR/BasicBlock.h"
168#include "llvm/IR/CallingConv.h"
169#include "llvm/IR/Constant.h"
170#include "llvm/IR/Constants.h"
171#include "llvm/IR/DataLayout.h"
172#include "llvm/IR/DerivedTypes.h"
173#include "llvm/IR/Function.h"
174#include "llvm/IR/GlobalValue.h"
175#include "llvm/IR/GlobalVariable.h"
176#include "llvm/IR/IRBuilder.h"
177#include "llvm/IR/InlineAsm.h"
178#include "llvm/IR/InstVisitor.h"
179#include "llvm/IR/InstrTypes.h"
180#include "llvm/IR/Instruction.h"
181#include "llvm/IR/Instructions.h"
182#include "llvm/IR/IntrinsicInst.h"
183#include "llvm/IR/Intrinsics.h"
184#include "llvm/IR/IntrinsicsAArch64.h"
185#include "llvm/IR/IntrinsicsX86.h"
186#include "llvm/IR/MDBuilder.h"
187#include "llvm/IR/Module.h"
188#include "llvm/IR/Type.h"
189#include "llvm/IR/Value.h"
190#include "llvm/IR/ValueMap.h"
191#include "llvm/Support/Alignment.h"
192#include "llvm/Support/AtomicOrdering.h"
193#include "llvm/Support/Casting.h"
194#include "llvm/Support/CommandLine.h"
195#include "llvm/Support/Debug.h"
196#include "llvm/Support/DebugCounter.h"
197#include "llvm/Support/ErrorHandling.h"
198#include "llvm/Support/MathExtras.h"
199#include "llvm/Support/raw_ostream.h"
200#include "llvm/TargetParser/Triple.h"
201#include "llvm/Transforms/Utils/BasicBlockUtils.h"
202#include "llvm/Transforms/Utils/Instrumentation.h"
203#include "llvm/Transforms/Utils/Local.h"
204#include "llvm/Transforms/Utils/ModuleUtils.h"
205#include <algorithm>
206#include <cassert>
207#include <cstddef>
208#include <cstdint>
209#include <memory>
210#include <numeric>
211#include <string>
212#include <tuple>
213
214using namespace llvm;
215
216#define DEBUG_TYPE "msan"
217
218DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
219 "Controls which checks to insert");
220
221DEBUG_COUNTER(DebugInstrumentInstruction, "msan-instrument-instruction",
222 "Controls which instruction to instrument");
223
224static const unsigned kOriginSize = 4;
225static const Align kMinOriginAlignment = Align(4);
226static const Align kShadowTLSAlignment = Align(8);
227
228// These constants must be kept in sync with the ones in msan.h.
229// TODO: increase size to match SVE/SVE2/SME/SME2 limits
230static const unsigned kParamTLSSize = 800;
231static const unsigned kRetvalTLSSize = 800;
232
233// Accesses sizes are powers of two: 1, 2, 4, 8.
234static const size_t kNumberOfAccessSizes = 4;
235
236/// Track origins of uninitialized values.
237///
238/// Adds a section to MemorySanitizer report that points to the allocation
239/// (stack or heap) the uninitialized bits came from originally.
240static cl::opt<int> ClTrackOrigins(
241 "msan-track-origins",
242 cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden,
243 cl::init(Val: 0));
244
245static cl::opt<bool> ClKeepGoing("msan-keep-going",
246 cl::desc("keep going after reporting a UMR"),
247 cl::Hidden, cl::init(Val: false));
248
249static cl::opt<bool>
250 ClPoisonStack("msan-poison-stack",
251 cl::desc("poison uninitialized stack variables"), cl::Hidden,
252 cl::init(Val: true));
253
254static cl::opt<bool> ClPoisonStackWithCall(
255 "msan-poison-stack-with-call",
256 cl::desc("poison uninitialized stack variables with a call"), cl::Hidden,
257 cl::init(Val: false));
258
259static cl::opt<int> ClPoisonStackPattern(
260 "msan-poison-stack-pattern",
261 cl::desc("poison uninitialized stack variables with the given pattern"),
262 cl::Hidden, cl::init(Val: 0xff));
263
264static cl::opt<bool>
265 ClPrintStackNames("msan-print-stack-names",
266 cl::desc("Print name of local stack variable"),
267 cl::Hidden, cl::init(Val: true));
268
269static cl::opt<bool>
270 ClPoisonUndef("msan-poison-undef",
271 cl::desc("Poison fully undef temporary values. "
272 "Partially undefined constant vectors "
273 "are unaffected by this flag (see "
274 "-msan-poison-undef-vectors)."),
275 cl::Hidden, cl::init(Val: true));
276
277static cl::opt<bool> ClPoisonUndefVectors(
278 "msan-poison-undef-vectors",
279 cl::desc("Precisely poison partially undefined constant vectors. "
280 "If false (legacy behavior), the entire vector is "
281 "considered fully initialized, which may lead to false "
282 "negatives. Fully undefined constant vectors are "
283 "unaffected by this flag (see -msan-poison-undef)."),
284 cl::Hidden, cl::init(Val: false));
285
286static cl::opt<bool> ClPreciseDisjointOr(
287 "msan-precise-disjoint-or",
288 cl::desc("Precisely poison disjoint OR. If false (legacy behavior), "
289 "disjointedness is ignored (i.e., 1|1 is initialized)."),
290 cl::Hidden, cl::init(Val: false));
291
292static cl::opt<bool>
293 ClHandleICmp("msan-handle-icmp",
294 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
295 cl::Hidden, cl::init(Val: true));
296
297static cl::opt<bool>
298 ClHandleICmpExact("msan-handle-icmp-exact",
299 cl::desc("exact handling of relational integer ICmp"),
300 cl::Hidden, cl::init(Val: true));
301
302static cl::opt<int> ClSwitchPrecision(
303 "msan-switch-precision",
304 cl::desc("Controls the number of cases considered by MSan for LLVM switch "
305 "instructions. 0 means no UUMs detected. Higher values lead to "
306 "fewer false negatives but may impact compiler and/or "
307 "application performance. N.B. LLVM switch instructions do not "
308 "correspond exactly to C++ switch statements."),
309 cl::Hidden, cl::init(Val: 99));
310
311static cl::opt<bool> ClHandleLifetimeIntrinsics(
312 "msan-handle-lifetime-intrinsics",
313 cl::desc(
314 "when possible, poison scoped variables at the beginning of the scope "
315 "(slower, but more precise)"),
316 cl::Hidden, cl::init(Val: true));
317
318// When compiling the Linux kernel, we sometimes see false positives related to
319// MSan being unable to understand that inline assembly calls may initialize
320// local variables.
321// This flag makes the compiler conservatively unpoison every memory location
322// passed into an assembly call. Note that this may cause false positives.
323// Because it's impossible to figure out the array sizes, we can only unpoison
324// the first sizeof(type) bytes for each type* pointer.
325static cl::opt<bool> ClHandleAsmConservative(
326 "msan-handle-asm-conservative",
327 cl::desc("conservative handling of inline assembly"), cl::Hidden,
328 cl::init(Val: true));
329
330// This flag controls whether we check the shadow of the address
331// operand of load or store. Such bugs are very rare, since load from
332// a garbage address typically results in SEGV, but still happen
333// (e.g. only lower bits of address are garbage, or the access happens
334// early at program startup where malloc-ed memory is more likely to
335// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
336static cl::opt<bool> ClCheckAccessAddress(
337 "msan-check-access-address",
338 cl::desc("report accesses through a pointer which has poisoned shadow"),
339 cl::Hidden, cl::init(Val: true));
340
341static cl::opt<bool> ClEagerChecks(
342 "msan-eager-checks",
343 cl::desc("check arguments and return values at function call boundaries"),
344 cl::Hidden, cl::init(Val: false));
345
346static cl::opt<bool> ClDumpStrictInstructions(
347 "msan-dump-strict-instructions",
348 cl::desc("print out instructions with default strict semantics i.e.,"
349 "check that all the inputs are fully initialized, and mark "
350 "the output as fully initialized. These semantics are applied "
351 "to instructions that could not be handled explicitly nor "
352 "heuristically."),
353 cl::Hidden, cl::init(Val: false));
354
355// Currently, all the heuristically handled instructions are specifically
356// IntrinsicInst. However, we use the broader "HeuristicInstructions" name
357// to parallel 'msan-dump-strict-instructions', and to keep the door open to
358// handling non-intrinsic instructions heuristically.
359static cl::opt<bool> ClDumpHeuristicInstructions(
360 "msan-dump-heuristic-instructions",
361 cl::desc("Prints 'unknown' instructions that were handled heuristically. "
362 "Use -msan-dump-strict-instructions to print instructions that "
363 "could not be handled explicitly nor heuristically."),
364 cl::Hidden, cl::init(Val: false));
365
366static cl::opt<int> ClInstrumentationWithCallThreshold(
367 "msan-instrumentation-with-call-threshold",
368 cl::desc(
369 "If the function being instrumented requires more than "
370 "this number of checks and origin stores, use callbacks instead of "
371 "inline checks (-1 means never use callbacks)."),
372 cl::Hidden, cl::init(Val: 3500));
373
374static cl::opt<bool>
375 ClEnableKmsan("msan-kernel",
376 cl::desc("Enable KernelMemorySanitizer instrumentation"),
377 cl::Hidden, cl::init(Val: false));
378
379static cl::opt<bool>
380 ClDisableChecks("msan-disable-checks",
381 cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
382 cl::init(Val: false));
383
384static cl::opt<bool>
385 ClCheckConstantShadow("msan-check-constant-shadow",
386 cl::desc("Insert checks for constant shadow values"),
387 cl::Hidden, cl::init(Val: true));
388
389// This is off by default because of a bug in gold:
390// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
391static cl::opt<bool>
392 ClWithComdat("msan-with-comdat",
393 cl::desc("Place MSan constructors in comdat sections"),
394 cl::Hidden, cl::init(Val: false));
395
396// These options allow to specify custom memory map parameters
397// See MemoryMapParams for details.
398static cl::opt<uint64_t> ClAndMask("msan-and-mask",
399 cl::desc("Define custom MSan AndMask"),
400 cl::Hidden, cl::init(Val: 0));
401
402static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
403 cl::desc("Define custom MSan XorMask"),
404 cl::Hidden, cl::init(Val: 0));
405
406static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
407 cl::desc("Define custom MSan ShadowBase"),
408 cl::Hidden, cl::init(Val: 0));
409
410static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
411 cl::desc("Define custom MSan OriginBase"),
412 cl::Hidden, cl::init(Val: 0));
413
414static cl::opt<int>
415 ClDisambiguateWarning("msan-disambiguate-warning-threshold",
416 cl::desc("Define threshold for number of checks per "
417 "debug location to force origin update."),
418 cl::Hidden, cl::init(Val: 3));
419
420const char kMsanModuleCtorName[] = "msan.module_ctor";
421const char kMsanInitName[] = "__msan_init";
422
423namespace {
424
425// Memory map parameters used in application-to-shadow address calculation.
426// Offset = (Addr & ~AndMask) ^ XorMask
427// Shadow = ShadowBase + Offset
428// Origin = OriginBase + Offset
429struct MemoryMapParams {
430 uint64_t AndMask;
431 uint64_t XorMask;
432 uint64_t ShadowBase;
433 uint64_t OriginBase;
434};
435
436struct PlatformMemoryMapParams {
437 const MemoryMapParams *bits32;
438 const MemoryMapParams *bits64;
439};
440
441} // end anonymous namespace
442
443// i386 Linux
444static const MemoryMapParams Linux_I386_MemoryMapParams = {
445 .AndMask: 0x000080000000, // AndMask
446 .XorMask: 0, // XorMask (not used)
447 .ShadowBase: 0, // ShadowBase (not used)
448 .OriginBase: 0x000040000000, // OriginBase
449};
450
451// x86_64 Linux
452static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
453 .AndMask: 0, // AndMask (not used)
454 .XorMask: 0x500000000000, // XorMask
455 .ShadowBase: 0, // ShadowBase (not used)
456 .OriginBase: 0x100000000000, // OriginBase
457};
458
459// mips32 Linux
460// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
461// after picking good constants
462
463// mips64 Linux
464static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
465 .AndMask: 0, // AndMask (not used)
466 .XorMask: 0x008000000000, // XorMask
467 .ShadowBase: 0, // ShadowBase (not used)
468 .OriginBase: 0x002000000000, // OriginBase
469};
470
471// ppc32 Linux
472// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
473// after picking good constants
474
475// ppc64 Linux
476static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
477 .AndMask: 0xE00000000000, // AndMask
478 .XorMask: 0x100000000000, // XorMask
479 .ShadowBase: 0x080000000000, // ShadowBase
480 .OriginBase: 0x1C0000000000, // OriginBase
481};
482
483// s390x Linux
484static const MemoryMapParams Linux_S390X_MemoryMapParams = {
485 .AndMask: 0xC00000000000, // AndMask
486 .XorMask: 0, // XorMask (not used)
487 .ShadowBase: 0x080000000000, // ShadowBase
488 .OriginBase: 0x1C0000000000, // OriginBase
489};
490
491// arm32 Linux
492// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
493// after picking good constants
494
495// aarch64 Linux
496static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
497 .AndMask: 0, // AndMask (not used)
498 .XorMask: 0x0B00000000000, // XorMask
499 .ShadowBase: 0, // ShadowBase (not used)
500 .OriginBase: 0x0200000000000, // OriginBase
501};
502
503// loongarch64 Linux
504static const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
505 .AndMask: 0, // AndMask (not used)
506 .XorMask: 0x500000000000, // XorMask
507 .ShadowBase: 0, // ShadowBase (not used)
508 .OriginBase: 0x100000000000, // OriginBase
509};
510
511// hexagon Linux
512static const MemoryMapParams Linux_Hexagon_MemoryMapParams = {
513 .AndMask: 0, // AndMask (not used)
514 .XorMask: 0x20000000, // XorMask
515 .ShadowBase: 0, // ShadowBase (not used)
516 .OriginBase: 0x50000000, // OriginBase
517};
518
519// riscv32 Linux
520// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
521// after picking good constants
522
523// aarch64 FreeBSD
524static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
525 .AndMask: 0x1800000000000, // AndMask
526 .XorMask: 0x0400000000000, // XorMask
527 .ShadowBase: 0x0200000000000, // ShadowBase
528 .OriginBase: 0x0700000000000, // OriginBase
529};
530
531// i386 FreeBSD
532static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
533 .AndMask: 0x000180000000, // AndMask
534 .XorMask: 0x000040000000, // XorMask
535 .ShadowBase: 0x000020000000, // ShadowBase
536 .OriginBase: 0x000700000000, // OriginBase
537};
538
539// x86_64 FreeBSD
540static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
541 .AndMask: 0xc00000000000, // AndMask
542 .XorMask: 0x200000000000, // XorMask
543 .ShadowBase: 0x100000000000, // ShadowBase
544 .OriginBase: 0x380000000000, // OriginBase
545};
546
547// x86_64 NetBSD
548static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
549 .AndMask: 0, // AndMask
550 .XorMask: 0x500000000000, // XorMask
551 .ShadowBase: 0, // ShadowBase
552 .OriginBase: 0x100000000000, // OriginBase
553};
554
555static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
556 .bits32: &Linux_I386_MemoryMapParams,
557 .bits64: &Linux_X86_64_MemoryMapParams,
558};
559
560static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
561 .bits32: nullptr,
562 .bits64: &Linux_MIPS64_MemoryMapParams,
563};
564
565static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
566 .bits32: nullptr,
567 .bits64: &Linux_PowerPC64_MemoryMapParams,
568};
569
570static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
571 .bits32: nullptr,
572 .bits64: &Linux_S390X_MemoryMapParams,
573};
574
575static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
576 .bits32: nullptr,
577 .bits64: &Linux_AArch64_MemoryMapParams,
578};
579
580static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams = {
581 .bits32: nullptr,
582 .bits64: &Linux_LoongArch64_MemoryMapParams,
583};
584
585static const PlatformMemoryMapParams Linux_Hexagon_MemoryMapParams_P = {
586 .bits32: &Linux_Hexagon_MemoryMapParams,
587 .bits64: nullptr,
588};
589
590static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
591 .bits32: nullptr,
592 .bits64: &FreeBSD_AArch64_MemoryMapParams,
593};
594
595static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
596 .bits32: &FreeBSD_I386_MemoryMapParams,
597 .bits64: &FreeBSD_X86_64_MemoryMapParams,
598};
599
600static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
601 .bits32: nullptr,
602 .bits64: &NetBSD_X86_64_MemoryMapParams,
603};
604
605enum OddOrEvenLanes { kBothLanes, kEvenLanes, kOddLanes };
606
607namespace {
608
609/// Instrument functions of a module to detect uninitialized reads.
610///
611/// Instantiating MemorySanitizer inserts the msan runtime library API function
612/// declarations into the module if they don't exist already. Instantiating
613/// ensures the __msan_init function is in the list of global constructors for
614/// the module.
615class MemorySanitizer {
616public:
617 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
618 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
619 Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
620 initializeModule(M);
621 }
622
623 // MSan cannot be moved or copied because of MapParams.
624 MemorySanitizer(MemorySanitizer &&) = delete;
625 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
626 MemorySanitizer(const MemorySanitizer &) = delete;
627 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
628
629 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
630
631private:
632 friend struct MemorySanitizerVisitor;
633 friend struct VarArgHelperBase;
634 friend struct VarArgAMD64Helper;
635 friend struct VarArgAArch64Helper;
636 friend struct VarArgPowerPC64Helper;
637 friend struct VarArgPowerPC32Helper;
638 friend struct VarArgSystemZHelper;
639 friend struct VarArgI386Helper;
640 friend struct VarArgGenericHelper;
641
642 void initializeModule(Module &M);
643 void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
644 void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
645 void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
646
647 template <typename... ArgsTy>
648 FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
649 ArgsTy... Args);
650
651 /// True if we're compiling the Linux kernel.
652 bool CompileKernel;
653 /// Track origins (allocation points) of uninitialized values.
654 int TrackOrigins;
655 bool Recover;
656 bool EagerChecks;
657
658 Triple TargetTriple;
659 LLVMContext *C;
660 Type *IntptrTy; ///< Integer type with the size of a ptr in default AS.
661 Type *OriginTy;
662 PointerType *PtrTy; ///< Integer type with the size of a ptr in default AS.
663
664 // XxxTLS variables represent the per-thread state in MSan and per-task state
665 // in KMSAN.
666 // For the userspace these point to thread-local globals. In the kernel land
667 // they point to the members of a per-task struct obtained via a call to
668 // __msan_get_context_state().
669
670 /// Thread-local shadow storage for function parameters.
671 Value *ParamTLS;
672
673 /// Thread-local origin storage for function parameters.
674 Value *ParamOriginTLS;
675
676 /// Thread-local shadow storage for function return value.
677 Value *RetvalTLS;
678
679 /// Thread-local origin storage for function return value.
680 Value *RetvalOriginTLS;
681
682 /// Thread-local shadow storage for in-register va_arg function.
683 Value *VAArgTLS;
684
685 /// Thread-local shadow storage for in-register va_arg function.
686 Value *VAArgOriginTLS;
687
688 /// Thread-local shadow storage for va_arg overflow area.
689 Value *VAArgOverflowSizeTLS;
690
691 /// Are the instrumentation callbacks set up?
692 bool CallbacksInitialized = false;
693
694 /// The run-time callback to print a warning.
695 FunctionCallee WarningFn;
696
697 // These arrays are indexed by log2(AccessSize).
698 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
699 FunctionCallee MaybeWarningVarSizeFn;
700 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
701
702 /// Run-time helper that generates a new origin value for a stack
703 /// allocation.
704 FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
705 // No description version
706 FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
707
708 /// Run-time helper that poisons stack on function entry.
709 FunctionCallee MsanPoisonStackFn;
710
711 /// Run-time helper that records a store (or any event) of an
712 /// uninitialized value and returns an updated origin id encoding this info.
713 FunctionCallee MsanChainOriginFn;
714
715 /// Run-time helper that paints an origin over a region.
716 FunctionCallee MsanSetOriginFn;
717
718 /// MSan runtime replacements for memmove, memcpy and memset.
719 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
720
721 /// KMSAN callback for task-local function argument shadow.
722 StructType *MsanContextStateTy;
723 FunctionCallee MsanGetContextStateFn;
724
725 /// Functions for poisoning/unpoisoning local variables
726 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
727
728 /// Pair of shadow/origin pointers.
729 Type *MsanMetadata;
730
731 /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
732 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
733 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
734 FunctionCallee MsanMetadataPtrForStore_1_8[4];
735 FunctionCallee MsanInstrumentAsmStoreFn;
736
737 /// Storage for return values of the MsanMetadataPtrXxx functions.
738 Value *MsanMetadataAlloca;
739
740 /// Helper to choose between different MsanMetadataPtrXxx().
741 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
742
743 /// Memory map parameters used in application-to-shadow calculation.
744 const MemoryMapParams *MapParams;
745
746 /// Custom memory map parameters used when -msan-shadow-base or
747 // -msan-origin-base is provided.
748 MemoryMapParams CustomMapParams;
749
750 MDNode *ColdCallWeights;
751
752 /// Branch weights for origin store.
753 MDNode *OriginStoreWeights;
754};
755
756void insertModuleCtor(Module &M) {
757 getOrCreateSanitizerCtorAndInitFunctions(
758 M, CtorName: kMsanModuleCtorName, InitName: kMsanInitName,
759 /*InitArgTypes=*/{},
760 /*InitArgs=*/{},
761 // This callback is invoked when the functions are created the first
762 // time. Hook them into the global ctors list in that case:
763 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
764 if (!ClWithComdat) {
765 appendToGlobalCtors(M, F: Ctor, Priority: 0);
766 return;
767 }
768 Comdat *MsanCtorComdat = M.getOrInsertComdat(Name: kMsanModuleCtorName);
769 Ctor->setComdat(MsanCtorComdat);
770 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
771 });
772}
773
774template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
775 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
776}
777
778} // end anonymous namespace
779
780MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K,
781 bool EagerChecks)
782 : Kernel(getOptOrDefault(Opt: ClEnableKmsan, Default: K)),
783 TrackOrigins(getOptOrDefault(Opt: ClTrackOrigins, Default: Kernel ? 2 : TO)),
784 Recover(getOptOrDefault(Opt: ClKeepGoing, Default: Kernel || R)),
785 EagerChecks(getOptOrDefault(Opt: ClEagerChecks, Default: EagerChecks)) {}
786
787PreservedAnalyses MemorySanitizerPass::run(Module &M,
788 ModuleAnalysisManager &AM) {
789 // Return early if nosanitize_memory module flag is present for the module.
790 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_memory"))
791 return PreservedAnalyses::all();
792 bool Modified = false;
793 if (!Options.Kernel) {
794 insertModuleCtor(M);
795 Modified = true;
796 }
797
798 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
799 for (Function &F : M) {
800 if (F.empty())
801 continue;
802 MemorySanitizer Msan(*F.getParent(), Options);
803 Modified |=
804 Msan.sanitizeFunction(F, TLI&: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
805 }
806
807 if (!Modified)
808 return PreservedAnalyses::all();
809
810 PreservedAnalyses PA = PreservedAnalyses::none();
811 // GlobalsAA is considered stateless and does not get invalidated unless
812 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
813 // make changes that require GlobalsAA to be invalidated.
814 PA.abandon<GlobalsAA>();
815 return PA;
816}
817
818void MemorySanitizerPass::printPipeline(
819 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
820 static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
821 OS, MapClassName2PassName);
822 OS << '<';
823 if (Options.Recover)
824 OS << "recover;";
825 if (Options.Kernel)
826 OS << "kernel;";
827 if (Options.EagerChecks)
828 OS << "eager-checks;";
829 OS << "track-origins=" << Options.TrackOrigins;
830 OS << '>';
831}
832
833/// Create a non-const global initialized with the given string.
834///
835/// Creates a writable global for Str so that we can pass it to the
836/// run-time lib. Runtime uses first 4 bytes of the string to store the
837/// frame ID, so the string needs to be mutable.
838static GlobalVariable *createPrivateConstGlobalForString(Module &M,
839 StringRef Str) {
840 Constant *StrConst = ConstantDataArray::getString(Context&: M.getContext(), Initializer: Str);
841 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
842 GlobalValue::PrivateLinkage, StrConst, "");
843}
844
845template <typename... ArgsTy>
846FunctionCallee
847MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
848 ArgsTy... Args) {
849 if (TargetTriple.getArch() == Triple::systemz) {
850 // SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
851 return M.getOrInsertFunction(Name, Type::getVoidTy(C&: *C), PtrTy,
852 std::forward<ArgsTy>(Args)...);
853 }
854
855 return M.getOrInsertFunction(Name, MsanMetadata,
856 std::forward<ArgsTy>(Args)...);
857}
858
859/// Create KMSAN API callbacks.
860void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
861 IRBuilder<> IRB(*C);
862
863 // These will be initialized in insertKmsanPrologue().
864 RetvalTLS = nullptr;
865 RetvalOriginTLS = nullptr;
866 ParamTLS = nullptr;
867 ParamOriginTLS = nullptr;
868 VAArgTLS = nullptr;
869 VAArgOriginTLS = nullptr;
870 VAArgOverflowSizeTLS = nullptr;
871
872 WarningFn = M.getOrInsertFunction(Name: "__msan_warning",
873 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
874 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
875
876 // Requests the per-task context state (kmsan_context_state*) from the
877 // runtime library.
878 MsanContextStateTy = StructType::get(
879 elt1: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
880 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8),
881 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
882 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8), /* va_arg_origin */
883 elts: IRB.getInt64Ty(), elts: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4), elts: OriginTy,
884 elts: OriginTy);
885 MsanGetContextStateFn =
886 M.getOrInsertFunction(Name: "__msan_get_context_state", RetTy: PtrTy);
887
888 MsanMetadata = StructType::get(elt1: PtrTy, elts: PtrTy);
889
890 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
891 std::string name_load =
892 "__msan_metadata_ptr_for_load_" + std::to_string(val: size);
893 std::string name_store =
894 "__msan_metadata_ptr_for_store_" + std::to_string(val: size);
895 MsanMetadataPtrForLoad_1_8[ind] =
896 getOrInsertMsanMetadataFunction(M, Name: name_load, Args: PtrTy);
897 MsanMetadataPtrForStore_1_8[ind] =
898 getOrInsertMsanMetadataFunction(M, Name: name_store, Args: PtrTy);
899 }
900
901 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
902 M, Name: "__msan_metadata_ptr_for_load_n", Args: PtrTy, Args: IntptrTy);
903 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
904 M, Name: "__msan_metadata_ptr_for_store_n", Args: PtrTy, Args: IntptrTy);
905
906 // Functions for poisoning and unpoisoning memory.
907 MsanPoisonAllocaFn = M.getOrInsertFunction(
908 Name: "__msan_poison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
909 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
910 Name: "__msan_unpoison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
911}
912
913static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
914 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&] {
915 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
916 nullptr, Name, nullptr,
917 GlobalVariable::InitialExecTLSModel);
918 });
919}
920
921/// Insert declarations for userspace-specific functions and globals.
922void MemorySanitizer::createUserspaceApi(Module &M,
923 const TargetLibraryInfo &TLI) {
924 IRBuilder<> IRB(*C);
925
926 // Create the callback.
927 // FIXME: this function should have "Cold" calling conv,
928 // which is not yet implemented.
929 if (TrackOrigins) {
930 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
931 : "__msan_warning_with_origin_noreturn";
932 WarningFn = M.getOrInsertFunction(Name: WarningFnName,
933 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
934 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
935 } else {
936 StringRef WarningFnName =
937 Recover ? "__msan_warning" : "__msan_warning_noreturn";
938 WarningFn = M.getOrInsertFunction(Name: WarningFnName, RetTy: IRB.getVoidTy());
939 }
940
941 // Create the global TLS variables.
942 RetvalTLS =
943 getOrInsertGlobal(M, Name: "__msan_retval_tls",
944 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8));
945
946 RetvalOriginTLS = getOrInsertGlobal(M, Name: "__msan_retval_origin_tls", Ty: OriginTy);
947
948 ParamTLS =
949 getOrInsertGlobal(M, Name: "__msan_param_tls",
950 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
951
952 ParamOriginTLS =
953 getOrInsertGlobal(M, Name: "__msan_param_origin_tls",
954 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
955
956 VAArgTLS =
957 getOrInsertGlobal(M, Name: "__msan_va_arg_tls",
958 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
959
960 VAArgOriginTLS =
961 getOrInsertGlobal(M, Name: "__msan_va_arg_origin_tls",
962 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
963
964 VAArgOverflowSizeTLS = getOrInsertGlobal(M, Name: "__msan_va_arg_overflow_size_tls",
965 Ty: IRB.getIntPtrTy(DL: M.getDataLayout()));
966
967 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
968 AccessSizeIndex++) {
969 unsigned AccessSize = 1 << AccessSizeIndex;
970 std::string FunctionName = "__msan_maybe_warning_" + itostr(X: AccessSize);
971 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
972 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 1}, /*Signed=*/false),
973 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: IRB.getInt32Ty());
974 MaybeWarningVarSizeFn = M.getOrInsertFunction(
975 Name: "__msan_maybe_warning_N", AttributeList: TLI.getAttrList(C, ArgNos: {}, /*Signed=*/false),
976 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IRB.getInt64Ty(), Args: IRB.getInt32Ty());
977 FunctionName = "__msan_maybe_store_origin_" + itostr(X: AccessSize);
978 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
979 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 2}, /*Signed=*/false),
980 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: PtrTy,
981 Args: IRB.getInt32Ty());
982 }
983
984 MsanSetAllocaOriginWithDescriptionFn =
985 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_with_descr",
986 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy, Args: PtrTy);
987 MsanSetAllocaOriginNoDescriptionFn =
988 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_no_descr",
989 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
990 MsanPoisonStackFn = M.getOrInsertFunction(Name: "__msan_poison_stack",
991 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
992}
993
994/// Insert extern declaration of runtime-provided functions and globals.
995void MemorySanitizer::initializeCallbacks(Module &M,
996 const TargetLibraryInfo &TLI) {
997 // Only do this once.
998 if (CallbacksInitialized)
999 return;
1000
1001 IRBuilder<> IRB(*C);
1002 // Initialize callbacks that are common for kernel and userspace
1003 // instrumentation.
1004 MsanChainOriginFn = M.getOrInsertFunction(
1005 Name: "__msan_chain_origin",
1006 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false, /*Ret=*/true), RetTy: IRB.getInt32Ty(),
1007 Args: IRB.getInt32Ty());
1008 MsanSetOriginFn = M.getOrInsertFunction(
1009 Name: "__msan_set_origin", AttributeList: TLI.getAttrList(C, ArgNos: {2}, /*Signed=*/false),
1010 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: IRB.getInt32Ty());
1011 MemmoveFn =
1012 M.getOrInsertFunction(Name: "__msan_memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
1013 MemcpyFn =
1014 M.getOrInsertFunction(Name: "__msan_memcpy", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
1015 MemsetFn = M.getOrInsertFunction(Name: "__msan_memset",
1016 AttributeList: TLI.getAttrList(C, ArgNos: {1}, /*Signed=*/true),
1017 RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
1018
1019 MsanInstrumentAsmStoreFn = M.getOrInsertFunction(
1020 Name: "__msan_instrument_asm_store", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
1021
1022 if (CompileKernel) {
1023 createKernelApi(M, TLI);
1024 } else {
1025 createUserspaceApi(M, TLI);
1026 }
1027 CallbacksInitialized = true;
1028}
1029
1030FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
1031 int size) {
1032 FunctionCallee *Fns =
1033 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
1034 switch (size) {
1035 case 1:
1036 return Fns[0];
1037 case 2:
1038 return Fns[1];
1039 case 4:
1040 return Fns[2];
1041 case 8:
1042 return Fns[3];
1043 default:
1044 return nullptr;
1045 }
1046}
1047
1048/// Module-level initialization.
1049///
1050/// inserts a call to __msan_init to the module's constructor list.
1051void MemorySanitizer::initializeModule(Module &M) {
1052 auto &DL = M.getDataLayout();
1053
1054 TargetTriple = M.getTargetTriple();
1055
1056 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
1057 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
1058 // Check the overrides first
1059 if (ShadowPassed || OriginPassed) {
1060 CustomMapParams.AndMask = ClAndMask;
1061 CustomMapParams.XorMask = ClXorMask;
1062 CustomMapParams.ShadowBase = ClShadowBase;
1063 CustomMapParams.OriginBase = ClOriginBase;
1064 MapParams = &CustomMapParams;
1065 } else {
1066 switch (TargetTriple.getOS()) {
1067 case Triple::FreeBSD:
1068 switch (TargetTriple.getArch()) {
1069 case Triple::aarch64:
1070 MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
1071 break;
1072 case Triple::x86_64:
1073 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
1074 break;
1075 case Triple::x86:
1076 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
1077 break;
1078 default:
1079 report_fatal_error(reason: "unsupported architecture");
1080 }
1081 break;
1082 case Triple::NetBSD:
1083 switch (TargetTriple.getArch()) {
1084 case Triple::x86_64:
1085 MapParams = NetBSD_X86_MemoryMapParams.bits64;
1086 break;
1087 default:
1088 report_fatal_error(reason: "unsupported architecture");
1089 }
1090 break;
1091 case Triple::Linux:
1092 switch (TargetTriple.getArch()) {
1093 case Triple::x86_64:
1094 MapParams = Linux_X86_MemoryMapParams.bits64;
1095 break;
1096 case Triple::x86:
1097 MapParams = Linux_X86_MemoryMapParams.bits32;
1098 break;
1099 case Triple::mips64:
1100 case Triple::mips64el:
1101 MapParams = Linux_MIPS_MemoryMapParams.bits64;
1102 break;
1103 case Triple::ppc64:
1104 case Triple::ppc64le:
1105 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
1106 break;
1107 case Triple::systemz:
1108 MapParams = Linux_S390_MemoryMapParams.bits64;
1109 break;
1110 case Triple::aarch64:
1111 case Triple::aarch64_be:
1112 MapParams = Linux_ARM_MemoryMapParams.bits64;
1113 break;
1114 case Triple::loongarch64:
1115 MapParams = Linux_LoongArch_MemoryMapParams.bits64;
1116 break;
1117 case Triple::hexagon:
1118 MapParams = Linux_Hexagon_MemoryMapParams_P.bits32;
1119 break;
1120 default:
1121 report_fatal_error(reason: "unsupported architecture");
1122 }
1123 break;
1124 default:
1125 report_fatal_error(reason: "unsupported operating system");
1126 }
1127 }
1128
1129 C = &(M.getContext());
1130 IRBuilder<> IRB(*C);
1131 IntptrTy = IRB.getIntPtrTy(DL);
1132 OriginTy = IRB.getInt32Ty();
1133 PtrTy = IRB.getPtrTy();
1134
1135 ColdCallWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1136 OriginStoreWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1137
1138 if (!CompileKernel) {
1139 if (TrackOrigins)
1140 M.getOrInsertGlobal(Name: "__msan_track_origins", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1141 return new GlobalVariable(
1142 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1143 IRB.getInt32(C: TrackOrigins), "__msan_track_origins");
1144 });
1145
1146 if (Recover)
1147 M.getOrInsertGlobal(Name: "__msan_keep_going", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1148 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1149 GlobalValue::WeakODRLinkage,
1150 IRB.getInt32(C: Recover), "__msan_keep_going");
1151 });
1152 }
1153}
1154
1155namespace {
1156
1157/// A helper class that handles instrumentation of VarArg
1158/// functions on a particular platform.
1159///
1160/// Implementations are expected to insert the instrumentation
1161/// necessary to propagate argument shadow through VarArg function
1162/// calls. Visit* methods are called during an InstVisitor pass over
1163/// the function, and should avoid creating new basic blocks. A new
1164/// instance of this class is created for each instrumented function.
1165struct VarArgHelper {
1166 virtual ~VarArgHelper() = default;
1167
1168 /// Visit a CallBase.
1169 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1170
1171 /// Visit a va_start call.
1172 virtual void visitVAStartInst(VAStartInst &I) = 0;
1173
1174 /// Visit a va_copy call.
1175 virtual void visitVACopyInst(VACopyInst &I) = 0;
1176
1177 /// Finalize function instrumentation.
1178 ///
1179 /// This method is called after visiting all interesting (see above)
1180 /// instructions in a function.
1181 virtual void finalizeInstrumentation() = 0;
1182};
1183
1184struct MemorySanitizerVisitor;
1185
1186} // end anonymous namespace
1187
1188static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1189 MemorySanitizerVisitor &Visitor);
1190
1191static unsigned TypeSizeToSizeIndex(TypeSize TS) {
1192 if (TS.isScalable())
1193 // Scalable types unconditionally take slowpaths.
1194 return kNumberOfAccessSizes;
1195 unsigned TypeSizeFixed = TS.getFixedValue();
1196 if (TypeSizeFixed <= 8)
1197 return 0;
1198 return Log2_32_Ceil(Value: (TypeSizeFixed + 7) / 8);
1199}
1200
1201namespace {
1202
1203/// Helper class to attach debug information of the given instruction onto new
1204/// instructions inserted after.
1205class NextNodeIRBuilder : public IRBuilder<> {
1206public:
1207 explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1208 SetCurrentDebugLocation(IP->getDebugLoc());
1209 }
1210};
1211
1212/// This class does all the work for a given function. Store and Load
1213/// instructions store and load corresponding shadow and origin
1214/// values. Most instructions propagate shadow from arguments to their
1215/// return values. Certain instructions (most importantly, BranchInst)
1216/// test their argument shadow and print reports (with a runtime call) if it's
1217/// non-zero.
1218struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1219 Function &F;
1220 MemorySanitizer &MS;
1221 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1222 ValueMap<Value *, Value *> ShadowMap, OriginMap;
1223 std::unique_ptr<VarArgHelper> VAHelper;
1224 const TargetLibraryInfo *TLI;
1225 Instruction *FnPrologueEnd;
1226 SmallVector<Instruction *, 16> Instructions;
1227
1228 // The following flags disable parts of MSan instrumentation based on
1229 // exclusion list contents and command-line options.
1230 bool InsertChecks;
1231 bool PropagateShadow;
1232 bool PoisonStack;
1233 bool PoisonUndef;
1234 bool PoisonUndefVectors;
1235
1236 struct ShadowOriginAndInsertPoint {
1237 Value *Shadow;
1238 Value *Origin;
1239 Instruction *OrigIns;
1240
1241 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1242 : Shadow(S), Origin(O), OrigIns(I) {}
1243 };
1244 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1245 DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
1246 SmallSetVector<AllocaInst *, 16> AllocaSet;
1247 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1248 SmallVector<StoreInst *, 16> StoreList;
1249 int64_t SplittableBlocksCount = 0;
1250
1251 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1252 const TargetLibraryInfo &TLI)
1253 : F(F), MS(MS), VAHelper(CreateVarArgHelper(Func&: F, Msan&: MS, Visitor&: *this)), TLI(&TLI) {
1254 bool SanitizeFunction =
1255 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) && !ClDisableChecks;
1256 InsertChecks = SanitizeFunction;
1257 PropagateShadow = SanitizeFunction;
1258 PoisonStack = SanitizeFunction && ClPoisonStack;
1259 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1260 PoisonUndefVectors = SanitizeFunction && ClPoisonUndefVectors;
1261
1262 // In the presence of unreachable blocks, we may see Phi nodes with
1263 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1264 // blocks, such nodes will not have any shadow value associated with them.
1265 // It's easier to remove unreachable blocks than deal with missing shadow.
1266 removeUnreachableBlocks(F);
1267
1268 MS.initializeCallbacks(M&: *F.getParent(), TLI);
1269 FnPrologueEnd =
1270 IRBuilder<>(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt())
1271 .CreateIntrinsicWithoutFolding(ID: Intrinsic::donothing, Args: {});
1272
1273 if (MS.CompileKernel) {
1274 IRBuilder<> IRB(FnPrologueEnd);
1275 insertKmsanPrologue(IRB);
1276 }
1277
1278 LLVM_DEBUG(if (!InsertChecks) dbgs()
1279 << "MemorySanitizer is not inserting checks into '"
1280 << F.getName() << "'\n");
1281 }
1282
1283 bool instrumentWithCalls(Value *V) {
1284 // Constants likely will be eliminated by follow-up passes.
1285 if (isa<Constant>(Val: V))
1286 return false;
1287 ++SplittableBlocksCount;
1288 return ClInstrumentationWithCallThreshold >= 0 &&
1289 SplittableBlocksCount > ClInstrumentationWithCallThreshold;
1290 }
1291
1292 bool isInPrologue(Instruction &I) {
1293 return I.getParent() == FnPrologueEnd->getParent() &&
1294 (&I == FnPrologueEnd || I.comesBefore(Other: FnPrologueEnd));
1295 }
1296
1297 // Creates a new origin and records the stack trace. In general we can call
1298 // this function for any origin manipulation we like. However it will cost
1299 // runtime resources. So use this wisely only if it can provide additional
1300 // information helpful to a user.
1301 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1302 if (MS.TrackOrigins <= 1)
1303 return V;
1304 return IRB.CreateCall(Callee: MS.MsanChainOriginFn, Args: V);
1305 }
1306
1307 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1308 const DataLayout &DL = F.getDataLayout();
1309 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1310 if (IntptrSize == kOriginSize)
1311 return Origin;
1312 assert(IntptrSize == kOriginSize * 2);
1313 Origin = IRB.CreateIntCast(V: Origin, DestTy: MS.IntptrTy, /* isSigned */ false);
1314 return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: kOriginSize * 8));
1315 }
1316
1317 /// Fill memory range with the given origin value.
1318 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1319 TypeSize TS, Align Alignment) {
1320 const DataLayout &DL = F.getDataLayout();
1321 const Align IntptrAlignment = DL.getABITypeAlign(Ty: MS.IntptrTy);
1322 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1323 assert(IntptrAlignment >= kMinOriginAlignment);
1324 assert(IntptrSize >= kOriginSize);
1325
1326 // Note: The loop based formation works for fixed length vectors too,
1327 // however we prefer to unroll and specialize alignment below.
1328 if (TS.isScalable()) {
1329 Value *Size = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size: TS);
1330 Value *RoundUp =
1331 IRB.CreateAdd(LHS: Size, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize - 1));
1332 Value *End =
1333 IRB.CreateUDiv(LHS: RoundUp, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize));
1334 auto [InsertPt, Index] =
1335 SplitBlockAndInsertSimpleForLoop(End, SplitBefore: IRB.GetInsertPoint());
1336 IRB.SetInsertPoint(InsertPt);
1337
1338 Value *GEP = IRB.CreateGEP(Ty: MS.OriginTy, Ptr: OriginPtr, IdxList: Index);
1339 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: kMinOriginAlignment);
1340 return;
1341 }
1342
1343 unsigned Size = TS.getFixedValue();
1344
1345 unsigned Ofs = 0;
1346 Align CurrentAlignment = Alignment;
1347 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1348 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1349 Value *IntptrOriginPtr = IRB.CreatePointerCast(V: OriginPtr, DestTy: MS.PtrTy);
1350 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1351 Value *Ptr = i ? IRB.CreateConstGEP1_32(Ty: MS.IntptrTy, Ptr: IntptrOriginPtr, Idx0: i)
1352 : IntptrOriginPtr;
1353 IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
1354 Ofs += IntptrSize / kOriginSize;
1355 CurrentAlignment = IntptrAlignment;
1356 }
1357 }
1358
1359 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1360 Value *GEP =
1361 i ? IRB.CreateConstGEP1_32(Ty: MS.OriginTy, Ptr: OriginPtr, Idx0: i) : OriginPtr;
1362 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
1363 CurrentAlignment = kMinOriginAlignment;
1364 }
1365 }
1366
1367 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1368 Value *OriginPtr, Align Alignment) {
1369 const DataLayout &DL = F.getDataLayout();
1370 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1371 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
1372 // ZExt cannot convert between vector and scalar
1373 Value *ConvertedShadow = convertShadowToScalar(V: Shadow, IRB);
1374 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1375 if (!ClCheckConstantShadow || ConstantShadow->isNullValue()) {
1376 // Origin is not needed: value is initialized or const shadow is
1377 // ignored.
1378 return;
1379 }
1380 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1381 // Copy origin as the value is definitely uninitialized.
1382 paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), OriginPtr, TS: StoreSize,
1383 Alignment: OriginAlignment);
1384 return;
1385 }
1386 // Fallback to runtime check, which still can be optimized out later.
1387 }
1388
1389 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1390 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1391 if (instrumentWithCalls(V: ConvertedShadow) &&
1392 SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1393 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1394 Value *ConvertedShadow2 =
1395 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1396 CallBase *CB = IRB.CreateCall(Callee: Fn, Args: {ConvertedShadow2, Addr, Origin});
1397 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1398 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1399 } else {
1400 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1401 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1402 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: MS.OriginStoreWeights);
1403 IRBuilder<> IRBNew(CheckTerm);
1404 paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), OriginPtr, TS: StoreSize,
1405 Alignment: OriginAlignment);
1406 }
1407 }
1408
1409 void materializeStores() {
1410 for (StoreInst *SI : StoreList) {
1411 IRBuilder<> IRB(SI);
1412 Value *Val = SI->getValueOperand();
1413 Value *Addr = SI->getPointerOperand();
1414 Value *Shadow = SI->isAtomic() ? getCleanShadow(V: Val) : getShadow(V: Val);
1415 Value *ShadowPtr, *OriginPtr;
1416 Type *ShadowTy = Shadow->getType();
1417 const Align Alignment = SI->getAlign();
1418 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1419 std::tie(args&: ShadowPtr, args&: OriginPtr) =
1420 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1421
1422 [[maybe_unused]] StoreInst *NewSI =
1423 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Alignment);
1424 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1425
1426 if (SI->isAtomic())
1427 SI->setOrdering(addReleaseOrdering(a: SI->getOrdering()));
1428
1429 if (MS.TrackOrigins && !SI->isAtomic())
1430 storeOrigin(IRB, Addr, Shadow, Origin: getOrigin(V: Val), OriginPtr,
1431 Alignment: OriginAlignment);
1432 }
1433 }
1434
1435 // Returns true if Debug Location corresponds to multiple warnings.
1436 bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1437 if (MS.TrackOrigins < 2)
1438 return false;
1439
1440 if (LazyWarningDebugLocationCount.empty())
1441 for (const auto &I : InstrumentationList)
1442 ++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1443
1444 return LazyWarningDebugLocationCount[DebugLoc] >= ClDisambiguateWarning;
1445 }
1446
1447 /// Helper function to insert a warning at IRB's current insert point.
1448 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1449 if (!Origin)
1450 Origin = (Value *)IRB.getInt32(C: 0);
1451 assert(Origin->getType()->isIntegerTy());
1452
1453 if (shouldDisambiguateWarningLocation(DebugLoc: IRB.getCurrentDebugLocation())) {
1454 // Try to create additional origin with debug info of the last origin
1455 // instruction. It may provide additional information to the user.
1456 if (Instruction *OI = dyn_cast_or_null<Instruction>(Val: Origin)) {
1457 assert(MS.TrackOrigins);
1458 auto NewDebugLoc = OI->getDebugLoc();
1459 // Origin update with missing or the same debug location provides no
1460 // additional value.
1461 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1462 // Insert update just before the check, so we call runtime only just
1463 // before the report.
1464 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1465 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1466 Origin = updateOrigin(V: Origin, IRB&: IRBOrigin);
1467 }
1468 }
1469 }
1470
1471 if (MS.CompileKernel || MS.TrackOrigins)
1472 IRB.CreateCall(Callee: MS.WarningFn, Args: Origin)->setCannotMerge();
1473 else
1474 IRB.CreateCall(Callee: MS.WarningFn)->setCannotMerge();
1475 // FIXME: Insert UnreachableInst if !MS.Recover?
1476 // This may invalidate some of the following checks and needs to be done
1477 // at the very end.
1478 }
1479
1480 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1481 Value *Origin) {
1482 const DataLayout &DL = F.getDataLayout();
1483 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1484 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1485 if (instrumentWithCalls(V: ConvertedShadow) && !MS.CompileKernel) {
1486 // ZExt cannot convert between vector and scalar
1487 ConvertedShadow = convertShadowToScalar(V: ConvertedShadow, IRB);
1488 Value *ConvertedShadow2 =
1489 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1490
1491 if (SizeIndex < kNumberOfAccessSizes) {
1492 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1493 CallBase *CB = IRB.CreateCall(
1494 Callee: Fn,
1495 Args: {ConvertedShadow2,
1496 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1497 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1498 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1499 } else {
1500 FunctionCallee Fn = MS.MaybeWarningVarSizeFn;
1501 Value *ShadowAlloca = IRB.CreateAlloca(Ty: ConvertedShadow2->getType(), AddrSpace: 0u);
1502 IRB.CreateStore(Val: ConvertedShadow2, Ptr: ShadowAlloca);
1503 unsigned ShadowSize = DL.getTypeAllocSize(Ty: ConvertedShadow2->getType());
1504 CallBase *CB = IRB.CreateCall(
1505 Callee: Fn,
1506 Args: {ShadowAlloca, ConstantInt::get(Ty: IRB.getInt64Ty(), V: ShadowSize),
1507 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1508 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1509 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1510 }
1511 } else {
1512 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1513 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1514 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(),
1515 /* Unreachable */ !MS.Recover, BranchWeights: MS.ColdCallWeights);
1516
1517 IRB.SetInsertPoint(CheckTerm);
1518 insertWarningFn(IRB, Origin);
1519 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1520 }
1521 }
1522
1523 void materializeInstructionChecks(
1524 ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1525 const DataLayout &DL = F.getDataLayout();
1526 // Disable combining in some cases. TrackOrigins checks each shadow to pick
1527 // correct origin.
1528 bool Combine = !MS.TrackOrigins;
1529 Instruction *Instruction = InstructionChecks.front().OrigIns;
1530 Value *Shadow = nullptr;
1531 for (const auto &ShadowData : InstructionChecks) {
1532 assert(ShadowData.OrigIns == Instruction);
1533 IRBuilder<> IRB(Instruction);
1534
1535 Value *ConvertedShadow = ShadowData.Shadow;
1536
1537 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1538 if (!ClCheckConstantShadow || ConstantShadow->isNullValue()) {
1539 // Skip, value is initialized or const shadow is ignored.
1540 continue;
1541 }
1542 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1543 // Report as the value is definitely uninitialized.
1544 insertWarningFn(IRB, Origin: ShadowData.Origin);
1545 if (!MS.Recover)
1546 return; // Always fail and stop here, not need to check the rest.
1547 // Skip entire instruction,
1548 continue;
1549 }
1550 // Fallback to runtime check, which still can be optimized out later.
1551 }
1552
1553 if (!Combine) {
1554 materializeOneCheck(IRB, ConvertedShadow, Origin: ShadowData.Origin);
1555 continue;
1556 }
1557
1558 if (!Shadow) {
1559 Shadow = ConvertedShadow;
1560 continue;
1561 }
1562
1563 Shadow = convertToBool(V: Shadow, IRB, name: "_mscmp");
1564 ConvertedShadow = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1565 Shadow = IRB.CreateOr(LHS: Shadow, RHS: ConvertedShadow, Name: "_msor");
1566 }
1567
1568 if (Shadow) {
1569 assert(Combine);
1570 IRBuilder<> IRB(Instruction);
1571 materializeOneCheck(IRB, ConvertedShadow: Shadow, Origin: nullptr);
1572 }
1573 }
1574
1575 static bool isAArch64SVCount(Type *Ty) {
1576 if (TargetExtType *TTy = dyn_cast<TargetExtType>(Val: Ty))
1577 return TTy->getName() == "aarch64.svcount";
1578 return false;
1579 }
1580
1581 // This is intended to match the "AArch64 Predicate-as-Counter Type" (aka
1582 // 'target("aarch64.svcount")', but not e.g., <vscale x 4 x i32>.
1583 static bool isScalableNonVectorType(Type *Ty) {
1584 if (!isAArch64SVCount(Ty))
1585 LLVM_DEBUG(dbgs() << "isScalableNonVectorType: Unexpected type " << *Ty
1586 << "\n");
1587
1588 return Ty->isScalableTy() && !isa<VectorType>(Val: Ty);
1589 }
1590
1591 void materializeChecks() {
1592#ifndef NDEBUG
1593 // For assert below.
1594 SmallPtrSet<Instruction *, 16> Done;
1595#endif
1596
1597 for (auto I = InstrumentationList.begin();
1598 I != InstrumentationList.end();) {
1599 auto OrigIns = I->OrigIns;
1600 // Checks are grouped by the original instruction. We call all
1601 // `insertShadowCheck` for an instruction at once.
1602 assert(Done.insert(OrigIns).second);
1603 auto J = std::find_if(first: I + 1, last: InstrumentationList.end(),
1604 pred: [OrigIns](const ShadowOriginAndInsertPoint &R) {
1605 return OrigIns != R.OrigIns;
1606 });
1607 // Process all checks of instruction at once.
1608 materializeInstructionChecks(InstructionChecks: ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1609 I = J;
1610 }
1611
1612 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1613 }
1614
1615 // Returns the last instruction in the new prologue
1616 void insertKmsanPrologue(IRBuilder<> &IRB) {
1617 Value *ContextState = IRB.CreateCall(Callee: MS.MsanGetContextStateFn, Args: {});
1618 Constant *Zero = IRB.getInt32(C: 0);
1619 MS.ParamTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1620 IdxList: {Zero, IRB.getInt32(C: 0)}, Name: "param_shadow");
1621 MS.RetvalTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1622 IdxList: {Zero, IRB.getInt32(C: 1)}, Name: "retval_shadow");
1623 MS.VAArgTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1624 IdxList: {Zero, IRB.getInt32(C: 2)}, Name: "va_arg_shadow");
1625 MS.VAArgOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1626 IdxList: {Zero, IRB.getInt32(C: 3)}, Name: "va_arg_origin");
1627 MS.VAArgOverflowSizeTLS =
1628 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1629 IdxList: {Zero, IRB.getInt32(C: 4)}, Name: "va_arg_overflow_size");
1630 MS.ParamOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1631 IdxList: {Zero, IRB.getInt32(C: 5)}, Name: "param_origin");
1632 MS.RetvalOriginTLS =
1633 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1634 IdxList: {Zero, IRB.getInt32(C: 6)}, Name: "retval_origin");
1635 if (MS.TargetTriple.getArch() == Triple::systemz)
1636 MS.MsanMetadataAlloca = IRB.CreateAlloca(Ty: MS.MsanMetadata, AddrSpace: 0u);
1637 }
1638
1639 /// Add MemorySanitizer instrumentation to a function.
1640 bool runOnFunction() {
1641 // Iterate all BBs in depth-first order and create shadow instructions
1642 // for all instructions (where applicable).
1643 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1644 for (BasicBlock *BB : depth_first(G: FnPrologueEnd->getParent()))
1645 visit(BB&: *BB);
1646
1647 // `visit` above only collects instructions. Process them after iterating
1648 // CFG to avoid requirement on CFG transformations.
1649 for (Instruction *I : Instructions)
1650 InstVisitor<MemorySanitizerVisitor>::visit(I&: *I);
1651
1652 // Finalize PHI nodes.
1653 for (PHINode *PN : ShadowPHINodes) {
1654 PHINode *PNS = cast<PHINode>(Val: getShadow(V: PN));
1655 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(Val: getOrigin(V: PN)) : nullptr;
1656 size_t NumValues = PN->getNumIncomingValues();
1657 for (size_t v = 0; v < NumValues; v++) {
1658 PNS->addIncoming(V: getShadow(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1659 if (PNO)
1660 PNO->addIncoming(V: getOrigin(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1661 }
1662 }
1663
1664 VAHelper->finalizeInstrumentation();
1665
1666 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1667 // instrumenting only allocas.
1668 if (ClHandleLifetimeIntrinsics) {
1669 for (auto Item : LifetimeStartList) {
1670 instrumentAlloca(I&: *Item.second, InsPoint: Item.first);
1671 AllocaSet.remove(X: Item.second);
1672 }
1673 }
1674 // Poison the allocas for which we didn't instrument the corresponding
1675 // lifetime intrinsics.
1676 for (AllocaInst *AI : AllocaSet)
1677 instrumentAlloca(I&: *AI);
1678
1679 // Insert shadow value checks.
1680 materializeChecks();
1681
1682 // Delayed instrumentation of StoreInst.
1683 // This may not add new address checks.
1684 materializeStores();
1685
1686 return true;
1687 }
1688
1689 /// Compute the shadow type that corresponds to a given Value.
1690 Type *getShadowTy(Value *V) { return getShadowTy(OrigTy: V->getType()); }
1691
1692 /// Compute the shadow type that corresponds to a given Type.
1693 Type *getShadowTy(Type *OrigTy) {
1694 if (!OrigTy->isSized()) {
1695 return nullptr;
1696 }
1697 // For integer type, shadow is the same as the original type.
1698 // This may return weird-sized types like i1.
1699 if (IntegerType *IT = dyn_cast<IntegerType>(Val: OrigTy))
1700 return IT;
1701 const DataLayout &DL = F.getDataLayout();
1702 if (VectorType *VT = dyn_cast<VectorType>(Val: OrigTy)) {
1703 uint32_t EltSize = DL.getTypeSizeInBits(Ty: VT->getElementType());
1704 return VectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSize),
1705 EC: VT->getElementCount());
1706 }
1707 if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy)) {
1708 return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1709 NumElements: AT->getNumElements());
1710 }
1711 if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1712 SmallVector<Type *, 4> Elements;
1713 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1714 Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: i)));
1715 StructType *Res = StructType::get(Context&: *MS.C, Elements, isPacked: ST->isPacked());
1716 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1717 return Res;
1718 }
1719 if (isScalableNonVectorType(Ty: OrigTy)) {
1720 LLVM_DEBUG(dbgs() << "getShadowTy: Scalable non-vector type: " << *OrigTy
1721 << "\n");
1722 return OrigTy;
1723 }
1724
1725 uint32_t TypeSize = DL.getTypeSizeInBits(Ty: OrigTy);
1726 return IntegerType::get(C&: *MS.C, NumBits: TypeSize);
1727 }
1728
1729 /// Extract combined shadow of struct elements as a bool
1730 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1731 IRBuilder<> &IRB) {
1732 Value *FalseVal = IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1733 Value *Aggregator = FalseVal;
1734
1735 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1736 // Combine by ORing together each element's bool shadow
1737 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1738 Value *ShadowBool = convertToBool(V: ShadowItem, IRB);
1739
1740 if (Aggregator != FalseVal)
1741 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowBool);
1742 else
1743 Aggregator = ShadowBool;
1744 }
1745
1746 return Aggregator;
1747 }
1748
1749 // Extract combined shadow of array elements
1750 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1751 IRBuilder<> &IRB) {
1752 if (!Array->getNumElements())
1753 return IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1754
1755 Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
1756 Value *Aggregator = convertShadowToScalar(V: FirstItem, IRB);
1757
1758 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1759 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1760 Value *ShadowInner = convertShadowToScalar(V: ShadowItem, IRB);
1761 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
1762 }
1763 return Aggregator;
1764 }
1765
1766 /// Convert a shadow value to it's flattened variant. The resulting
1767 /// shadow may not necessarily have the same bit width as the input
1768 /// value, but it will always be comparable to zero.
1769 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1770 if (StructType *Struct = dyn_cast<StructType>(Val: V->getType()))
1771 return collapseStructShadow(Struct, Shadow: V, IRB);
1772 if (ArrayType *Array = dyn_cast<ArrayType>(Val: V->getType()))
1773 return collapseArrayShadow(Array, Shadow: V, IRB);
1774 if (isa<VectorType>(Val: V->getType())) {
1775 if (isa<ScalableVectorType>(Val: V->getType()))
1776 return convertShadowToScalar(V: IRB.CreateOrReduce(Src: V), IRB);
1777 unsigned BitWidth =
1778 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1779 return IRB.CreateBitCast(V, DestTy: IntegerType::get(C&: *MS.C, NumBits: BitWidth));
1780 }
1781 return V;
1782 }
1783
1784 // Convert a scalar value to an i1 by comparing with 0
1785 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1786 Type *VTy = V->getType();
1787 if (!VTy->isIntegerTy())
1788 return convertToBool(V: convertShadowToScalar(V, IRB), IRB, name);
1789 if (VTy->getIntegerBitWidth() == 1)
1790 // Just converting a bool to a bool, so do nothing.
1791 return V;
1792 return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name: name);
1793 }
1794
1795 Type *ptrToIntPtrType(Type *PtrTy) const {
1796 if (VectorType *VectTy = dyn_cast<VectorType>(Val: PtrTy)) {
1797 return VectorType::get(ElementType: ptrToIntPtrType(PtrTy: VectTy->getElementType()),
1798 EC: VectTy->getElementCount());
1799 }
1800 assert(PtrTy->isIntOrPtrTy());
1801 return MS.IntptrTy;
1802 }
1803
1804 Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1805 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1806 return VectorType::get(
1807 ElementType: getPtrToShadowPtrType(IntPtrTy: VectTy->getElementType(), ShadowTy),
1808 EC: VectTy->getElementCount());
1809 }
1810 assert(IntPtrTy == MS.IntptrTy);
1811 return MS.PtrTy;
1812 }
1813
1814 Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1815 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1816 return ConstantVector::getSplat(
1817 EC: VectTy->getElementCount(),
1818 Elt: constToIntPtr(IntPtrTy: VectTy->getElementType(), C));
1819 }
1820 assert(IntPtrTy == MS.IntptrTy);
1821 // TODO: Avoid implicit trunc?
1822 // See https://github.com/llvm/llvm-project/issues/112510.
1823 return ConstantInt::get(Ty: MS.IntptrTy, V: C, /*IsSigned=*/false,
1824 /*ImplicitTrunc=*/true);
1825 }
1826
1827 /// Returns the integer shadow offset that corresponds to a given
1828 /// application address, whereby:
1829 ///
1830 /// Offset = (Addr & ~AndMask) ^ XorMask
1831 /// Shadow = ShadowBase + Offset
1832 /// Origin = (OriginBase + Offset) & ~Alignment
1833 ///
1834 /// Note: for efficiency, many shadow mappings only require use the XorMask
1835 /// and OriginBase; the AndMask and ShadowBase are often zero.
1836 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1837 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1838 Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1839
1840 if (uint64_t AndMask = MS.MapParams->AndMask)
1841 OffsetLong = IRB.CreateAnd(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~AndMask));
1842
1843 if (uint64_t XorMask = MS.MapParams->XorMask)
1844 OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: XorMask));
1845 return OffsetLong;
1846 }
1847
1848 /// Compute the shadow and origin addresses corresponding to a given
1849 /// application address.
1850 ///
1851 /// Shadow = ShadowBase + Offset
1852 /// Origin = (OriginBase + Offset) & ~3ULL
1853 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1854 /// a single pointee.
1855 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1856 std::pair<Value *, Value *>
1857 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1858 MaybeAlign Alignment) {
1859 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1860 if (!VectTy) {
1861 assert(Addr->getType()->isPointerTy());
1862 } else {
1863 assert(VectTy->getElementType()->isPointerTy());
1864 }
1865 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1866 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1867 Value *ShadowLong = ShadowOffset;
1868 if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1869 ShadowLong =
1870 IRB.CreateAdd(LHS: ShadowLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ShadowBase));
1871 }
1872 Value *ShadowPtr = IRB.CreateIntToPtr(
1873 V: ShadowLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy));
1874
1875 Value *OriginPtr = nullptr;
1876 if (MS.TrackOrigins) {
1877 Value *OriginLong = ShadowOffset;
1878 uint64_t OriginBase = MS.MapParams->OriginBase;
1879 if (OriginBase != 0)
1880 OriginLong =
1881 IRB.CreateAdd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: OriginBase));
1882 if (!Alignment || *Alignment < kMinOriginAlignment) {
1883 uint64_t Mask = kMinOriginAlignment.value() - 1;
1884 OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~Mask));
1885 }
1886 OriginPtr = IRB.CreateIntToPtr(
1887 V: OriginLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy: MS.OriginTy));
1888 }
1889 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1890 }
1891
1892 template <typename... ArgsTy>
1893 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1894 ArgsTy... Args) {
1895 if (MS.TargetTriple.getArch() == Triple::systemz) {
1896 IRB.CreateCall(Callee,
1897 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1898 return IRB.CreateLoad(Ty: MS.MsanMetadata, Ptr: MS.MsanMetadataAlloca);
1899 }
1900
1901 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1902 }
1903
1904 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
1905 IRBuilder<> &IRB,
1906 Type *ShadowTy,
1907 bool isStore) {
1908 Value *ShadowOriginPtrs;
1909 const DataLayout &DL = F.getDataLayout();
1910 TypeSize Size = DL.getTypeStoreSize(Ty: ShadowTy);
1911
1912 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, size: Size);
1913 Value *AddrCast = IRB.CreatePointerCast(V: Addr, DestTy: MS.PtrTy);
1914 if (Getter) {
1915 ShadowOriginPtrs = createMetadataCall(IRB, Callee: Getter, Args: AddrCast);
1916 } else {
1917 Value *SizeVal = ConstantInt::get(Ty: MS.IntptrTy, V: Size);
1918 ShadowOriginPtrs = createMetadataCall(
1919 IRB,
1920 Callee: isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1921 Args: AddrCast, Args: SizeVal);
1922 }
1923 Value *ShadowPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 0);
1924 ShadowPtr = IRB.CreatePointerCast(V: ShadowPtr, DestTy: MS.PtrTy);
1925 Value *OriginPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 1);
1926
1927 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1928 }
1929
1930 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1931 /// a single pointee.
1932 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1933 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1934 IRBuilder<> &IRB,
1935 Type *ShadowTy,
1936 bool isStore) {
1937 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1938 if (!VectTy) {
1939 assert(Addr->getType()->isPointerTy());
1940 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1941 }
1942
1943 // TODO: Support callbacs with vectors of addresses.
1944 unsigned NumElements = cast<FixedVectorType>(Val: VectTy)->getNumElements();
1945 Value *ShadowPtrs = ConstantInt::getNullValue(
1946 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1947 Value *OriginPtrs = nullptr;
1948 if (MS.TrackOrigins)
1949 OriginPtrs = ConstantInt::getNullValue(
1950 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1951 for (unsigned i = 0; i < NumElements; ++i) {
1952 Value *OneAddr =
1953 IRB.CreateExtractElement(Vec: Addr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1954 auto [ShadowPtr, OriginPtr] =
1955 getShadowOriginPtrKernelNoVec(Addr: OneAddr, IRB, ShadowTy, isStore);
1956
1957 ShadowPtrs = IRB.CreateInsertElement(
1958 Vec: ShadowPtrs, NewElt: ShadowPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1959 if (MS.TrackOrigins)
1960 OriginPtrs = IRB.CreateInsertElement(
1961 Vec: OriginPtrs, NewElt: OriginPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1962 }
1963 return {ShadowPtrs, OriginPtrs};
1964 }
1965
1966 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1967 Type *ShadowTy,
1968 MaybeAlign Alignment,
1969 bool isStore) {
1970 if (MS.CompileKernel)
1971 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1972 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1973 }
1974
1975 /// Compute the shadow address for a given function argument.
1976 ///
1977 /// Shadow = ParamTLS+ArgOffset.
1978 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1979 return IRB.CreatePtrAdd(Ptr: MS.ParamTLS,
1980 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg");
1981 }
1982
1983 /// Compute the origin address for a given function argument.
1984 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1985 if (!MS.TrackOrigins)
1986 return nullptr;
1987 return IRB.CreatePtrAdd(Ptr: MS.ParamOriginTLS,
1988 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
1989 Name: "_msarg_o");
1990 }
1991
1992 /// Compute the shadow address for a retval.
1993 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1994 return IRB.CreatePointerCast(V: MS.RetvalTLS, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msret");
1995 }
1996
1997 /// Compute the origin address for a retval.
1998 Value *getOriginPtrForRetval() {
1999 // We keep a single origin for the entire retval. Might be too optimistic.
2000 return MS.RetvalOriginTLS;
2001 }
2002
2003 /// Set SV to be the shadow value for V.
2004 void setShadow(Value *V, Value *SV) {
2005 assert(!ShadowMap.count(V) && "Values may only have one shadow");
2006 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
2007 }
2008
2009 /// Set Origin to be the origin value for V.
2010 void setOrigin(Value *V, Value *Origin) {
2011 if (!MS.TrackOrigins)
2012 return;
2013 assert(!OriginMap.count(V) && "Values may only have one origin");
2014 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
2015 OriginMap[V] = Origin;
2016 }
2017
2018 Constant *getCleanShadow(Type *OrigTy) {
2019 Type *ShadowTy = getShadowTy(OrigTy);
2020 if (!ShadowTy)
2021 return nullptr;
2022 return Constant::getNullValue(Ty: ShadowTy);
2023 }
2024
2025 /// Create a clean shadow value for a given value.
2026 ///
2027 /// Clean shadow (all zeroes) means all bits of the value are defined
2028 /// (initialized).
2029 Constant *getCleanShadow(Value *V) { return getCleanShadow(OrigTy: V->getType()); }
2030
2031 /// Create a dirty shadow of a given shadow type.
2032 Constant *getPoisonedShadow(Type *ShadowTy) {
2033 assert(ShadowTy);
2034 if (isa<IntegerType>(Val: ShadowTy) || isa<VectorType>(Val: ShadowTy))
2035 return Constant::getAllOnesValue(Ty: ShadowTy);
2036 if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy)) {
2037 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
2038 getPoisonedShadow(ShadowTy: AT->getElementType()));
2039 return ConstantArray::get(T: AT, V: Vals);
2040 }
2041 if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy)) {
2042 SmallVector<Constant *, 4> Vals;
2043 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
2044 Vals.push_back(Elt: getPoisonedShadow(ShadowTy: ST->getElementType(N: i)));
2045 return ConstantStruct::get(T: ST, V: Vals);
2046 }
2047 llvm_unreachable("Unexpected shadow type");
2048 }
2049
2050 /// Create a dirty shadow for a given value.
2051 Constant *getPoisonedShadow(Value *V) {
2052 Type *ShadowTy = getShadowTy(V);
2053 if (!ShadowTy)
2054 return nullptr;
2055 return getPoisonedShadow(ShadowTy);
2056 }
2057
2058 /// Create a clean (zero) origin.
2059 Value *getCleanOrigin() { return Constant::getNullValue(Ty: MS.OriginTy); }
2060
2061 /// Get the shadow value for a given Value.
2062 ///
2063 /// This function either returns the value set earlier with setShadow,
2064 /// or extracts if from ParamTLS (for function arguments).
2065 Value *getShadow(Value *V) {
2066 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2067 if (!PropagateShadow || I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2068 return getCleanShadow(V);
2069 // For instructions the shadow is already stored in the map.
2070 Value *Shadow = ShadowMap[V];
2071 if (!Shadow) {
2072 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
2073 assert(Shadow && "No shadow for a value");
2074 }
2075 return Shadow;
2076 }
2077 // Handle fully undefined values
2078 // (partially undefined constant vectors are handled later)
2079 if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(Val: V)) {
2080 Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
2081 : getCleanShadow(V);
2082 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
2083 return AllOnes;
2084 }
2085 if (Argument *A = dyn_cast<Argument>(Val: V)) {
2086 // For arguments we compute the shadow on demand and store it in the map.
2087 Value *&ShadowPtr = ShadowMap[V];
2088 if (ShadowPtr)
2089 return ShadowPtr;
2090 Function *F = A->getParent();
2091 IRBuilder<> EntryIRB(FnPrologueEnd);
2092 unsigned ArgOffset = 0;
2093 const DataLayout &DL = F->getDataLayout();
2094 for (auto &FArg : F->args()) {
2095 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
2096 LLVM_DEBUG(dbgs() << (FArg.getType()->isScalableTy()
2097 ? "vscale not fully supported\n"
2098 : "Arg is not sized\n"));
2099 if (A == &FArg) {
2100 ShadowPtr = getCleanShadow(V);
2101 setOrigin(V: A, Origin: getCleanOrigin());
2102 break;
2103 }
2104 continue;
2105 }
2106
2107 unsigned Size = FArg.hasByValAttr()
2108 ? DL.getTypeAllocSize(Ty: FArg.getParamByValType())
2109 : DL.getTypeAllocSize(Ty: FArg.getType());
2110
2111 if (A == &FArg) {
2112 bool Overflow = ArgOffset + Size > kParamTLSSize;
2113 if (FArg.hasByValAttr()) {
2114 // ByVal pointer itself has clean shadow. We copy the actual
2115 // argument shadow to the underlying memory.
2116 // Figure out maximal valid memcpy alignment.
2117 const Align ArgAlign = DL.getValueOrABITypeAlignment(
2118 Alignment: FArg.getParamAlign(), Ty: FArg.getParamByValType());
2119 Value *CpShadowPtr, *CpOriginPtr;
2120 std::tie(args&: CpShadowPtr, args&: CpOriginPtr) =
2121 getShadowOriginPtr(Addr: V, IRB&: EntryIRB, ShadowTy: EntryIRB.getInt8Ty(), Alignment: ArgAlign,
2122 /*isStore*/ true);
2123 if (!PropagateShadow || Overflow) {
2124 // ParamTLS overflow.
2125 EntryIRB.CreateMemSet(
2126 Ptr: CpShadowPtr, Val: Constant::getNullValue(Ty: EntryIRB.getInt8Ty()),
2127 Size, Align: ArgAlign);
2128 } else {
2129 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
2130 const Align CopyAlign = std::min(a: ArgAlign, b: kShadowTLSAlignment);
2131 [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
2132 Dst: CpShadowPtr, DstAlign: CopyAlign, Src: Base, SrcAlign: CopyAlign, Size);
2133 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
2134
2135 if (MS.TrackOrigins) {
2136 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
2137 // FIXME: OriginSize should be:
2138 // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
2139 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
2140 EntryIRB.CreateMemCpy(
2141 Dst: CpOriginPtr,
2142 /* by getShadowOriginPtr */ DstAlign: kMinOriginAlignment, Src: OriginPtr,
2143 /* by origin_tls[ArgOffset] */ SrcAlign: kMinOriginAlignment,
2144 Size: OriginSize);
2145 }
2146 }
2147 }
2148
2149 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2150 (MS.EagerChecks && FArg.hasAttribute(Kind: Attribute::NoUndef))) {
2151 ShadowPtr = getCleanShadow(V);
2152 setOrigin(V: A, Origin: getCleanOrigin());
2153 } else {
2154 // Shadow over TLS
2155 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
2156 ShadowPtr = EntryIRB.CreateAlignedLoad(Ty: getShadowTy(V: &FArg), Ptr: Base,
2157 Align: kShadowTLSAlignment);
2158 if (MS.TrackOrigins) {
2159 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
2160 setOrigin(V: A, Origin: EntryIRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
2161 }
2162 }
2163 LLVM_DEBUG(dbgs()
2164 << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
2165 break;
2166 }
2167
2168 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
2169 }
2170 assert(ShadowPtr && "Could not find shadow for an argument");
2171 return ShadowPtr;
2172 }
2173
2174 // Check for partially-undefined constant vectors
2175 // TODO: scalable vectors (this is hard because we do not have IRBuilder)
2176 if (isa<FixedVectorType>(Val: V->getType()) && isa<Constant>(Val: V) &&
2177 cast<Constant>(Val: V)->containsUndefOrPoisonElement() && PropagateShadow &&
2178 PoisonUndefVectors) {
2179 unsigned NumElems = cast<FixedVectorType>(Val: V->getType())->getNumElements();
2180 SmallVector<Constant *, 32> ShadowVector(NumElems);
2181 for (unsigned i = 0; i != NumElems; ++i) {
2182 Constant *Elem = cast<Constant>(Val: V)->getAggregateElement(Elt: i);
2183 ShadowVector[i] = isa<UndefValue>(Val: Elem) ? getPoisonedShadow(V: Elem)
2184 : getCleanShadow(V: Elem);
2185 }
2186
2187 Value *ShadowConstant = ConstantVector::get(V: ShadowVector);
2188 LLVM_DEBUG(dbgs() << "Partial undef constant vector: " << *V << " ==> "
2189 << *ShadowConstant << "\n");
2190
2191 return ShadowConstant;
2192 }
2193
2194 // TODO: partially-undefined constant arrays, structures, and nested types
2195
2196 // For everything else the shadow is zero.
2197 return getCleanShadow(V);
2198 }
2199
2200 /// Get the shadow for i-th argument of the instruction I.
2201 Value *getShadow(Instruction *I, int i) {
2202 return getShadow(V: I->getOperand(i));
2203 }
2204
2205 /// Get the origin for a value.
2206 Value *getOrigin(Value *V) {
2207 if (!MS.TrackOrigins)
2208 return nullptr;
2209 if (!PropagateShadow || isa<Constant>(Val: V) || isa<InlineAsm>(Val: V))
2210 return getCleanOrigin();
2211 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2212 "Unexpected value type in getOrigin()");
2213 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2214 if (I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2215 return getCleanOrigin();
2216 }
2217 Value *Origin = OriginMap[V];
2218 assert(Origin && "Missing origin");
2219 return Origin;
2220 }
2221
2222 /// Get the origin for i-th argument of the instruction I.
2223 Value *getOrigin(Instruction *I, int i) {
2224 return getOrigin(V: I->getOperand(i));
2225 }
2226
2227 /// Remember the place where a shadow check should be inserted.
2228 ///
2229 /// This location will be later instrumented with a check that will print a
2230 /// UMR warning in runtime if the shadow value is not 0.
2231 void insertCheckShadow(Value *Shadow, Value *Origin, Instruction *OrigIns) {
2232 assert(Shadow);
2233 if (!InsertChecks)
2234 return;
2235
2236 if (!DebugCounter::shouldExecute(Counter&: DebugInsertCheck)) {
2237 LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
2238 << *OrigIns << "\n");
2239 return;
2240 }
2241
2242 Type *ShadowTy = Shadow->getType();
2243 if (isScalableNonVectorType(Ty: ShadowTy)) {
2244 LLVM_DEBUG(dbgs() << "Skipping check of scalable non-vector " << *Shadow
2245 << " before " << *OrigIns << "\n");
2246 return;
2247 }
2248#ifndef NDEBUG
2249 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2250 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2251 "Can only insert checks for integer, vector, and aggregate shadow "
2252 "types");
2253#endif
2254 InstrumentationList.push_back(
2255 Elt: ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2256 }
2257
2258 /// Get shadow for value, and remember the place where a shadow check should
2259 /// be inserted.
2260 ///
2261 /// This location will be later instrumented with a check that will print a
2262 /// UMR warning in runtime if the value is not fully defined.
2263 void insertCheckShadowOf(Value *Val, Instruction *OrigIns) {
2264 assert(Val);
2265 Value *Shadow, *Origin;
2266 if (ClCheckConstantShadow) {
2267 Shadow = getShadow(V: Val);
2268 if (!Shadow)
2269 return;
2270 Origin = getOrigin(V: Val);
2271 } else {
2272 Shadow = dyn_cast_or_null<Instruction>(Val: getShadow(V: Val));
2273 if (!Shadow)
2274 return;
2275 Origin = dyn_cast_or_null<Instruction>(Val: getOrigin(V: Val));
2276 }
2277 insertCheckShadow(Shadow, Origin, OrigIns);
2278 }
2279
2280 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
2281 switch (a) {
2282 case AtomicOrdering::NotAtomic:
2283 return AtomicOrdering::NotAtomic;
2284 case AtomicOrdering::Unordered:
2285 case AtomicOrdering::Monotonic:
2286 case AtomicOrdering::Release:
2287 return AtomicOrdering::Release;
2288 case AtomicOrdering::Acquire:
2289 case AtomicOrdering::AcquireRelease:
2290 return AtomicOrdering::AcquireRelease;
2291 case AtomicOrdering::SequentiallyConsistent:
2292 return AtomicOrdering::SequentiallyConsistent;
2293 }
2294 llvm_unreachable("Unknown ordering");
2295 }
2296
2297 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2298 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2299 uint32_t OrderingTable[NumOrderings] = {};
2300
2301 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2302 OrderingTable[(int)AtomicOrderingCABI::release] =
2303 (int)AtomicOrderingCABI::release;
2304 OrderingTable[(int)AtomicOrderingCABI::consume] =
2305 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2306 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2307 (int)AtomicOrderingCABI::acq_rel;
2308 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2309 (int)AtomicOrderingCABI::seq_cst;
2310
2311 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2312 }
2313
2314 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
2315 switch (a) {
2316 case AtomicOrdering::NotAtomic:
2317 return AtomicOrdering::NotAtomic;
2318 case AtomicOrdering::Unordered:
2319 case AtomicOrdering::Monotonic:
2320 case AtomicOrdering::Acquire:
2321 return AtomicOrdering::Acquire;
2322 case AtomicOrdering::Release:
2323 case AtomicOrdering::AcquireRelease:
2324 return AtomicOrdering::AcquireRelease;
2325 case AtomicOrdering::SequentiallyConsistent:
2326 return AtomicOrdering::SequentiallyConsistent;
2327 }
2328 llvm_unreachable("Unknown ordering");
2329 }
2330
2331 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2332 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2333 uint32_t OrderingTable[NumOrderings] = {};
2334
2335 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2336 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2337 OrderingTable[(int)AtomicOrderingCABI::consume] =
2338 (int)AtomicOrderingCABI::acquire;
2339 OrderingTable[(int)AtomicOrderingCABI::release] =
2340 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2341 (int)AtomicOrderingCABI::acq_rel;
2342 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2343 (int)AtomicOrderingCABI::seq_cst;
2344
2345 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2346 }
2347
2348 // ------------------- Visitors.
2349 using InstVisitor<MemorySanitizerVisitor>::visit;
2350 void visit(Instruction &I) {
2351 if (I.getMetadata(KindID: LLVMContext::MD_nosanitize))
2352 return;
2353 // Don't want to visit if we're in the prologue
2354 if (isInPrologue(I))
2355 return;
2356 if (!DebugCounter::shouldExecute(Counter&: DebugInstrumentInstruction)) {
2357 LLVM_DEBUG(dbgs() << "Skipping instruction: " << I << "\n");
2358 // We still need to set the shadow and origin to clean values.
2359 setShadow(V: &I, SV: getCleanShadow(V: &I));
2360 setOrigin(V: &I, Origin: getCleanOrigin());
2361 return;
2362 }
2363
2364 Instructions.push_back(Elt: &I);
2365 }
2366
2367 /// Instrument LoadInst
2368 ///
2369 /// Loads the corresponding shadow and (optionally) origin.
2370 /// Optionally, checks that the load address is fully defined.
2371 void visitLoadInst(LoadInst &I) {
2372 assert(I.getType()->isSized() && "Load type must have size");
2373 assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2374 NextNodeIRBuilder IRB(&I);
2375 Type *ShadowTy = getShadowTy(V: &I);
2376 Value *Addr = I.getPointerOperand();
2377 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2378 const Align Alignment = I.getAlign();
2379 if (PropagateShadow) {
2380 std::tie(args&: ShadowPtr, args&: OriginPtr) =
2381 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2382 setShadow(V: &I,
2383 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2384 } else {
2385 setShadow(V: &I, SV: getCleanShadow(V: &I));
2386 }
2387
2388 if (ClCheckAccessAddress)
2389 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2390
2391 if (I.isAtomic())
2392 I.setOrdering(addAcquireOrdering(a: I.getOrdering()));
2393
2394 if (MS.TrackOrigins) {
2395 if (PropagateShadow) {
2396 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
2397 setOrigin(
2398 V: &I, Origin: IRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: OriginPtr, Align: OriginAlignment));
2399 } else {
2400 setOrigin(V: &I, Origin: getCleanOrigin());
2401 }
2402 }
2403 }
2404
2405 /// Instrument StoreInst
2406 ///
2407 /// Stores the corresponding shadow and (optionally) origin.
2408 /// Optionally, checks that the store address is fully defined.
2409 void visitStoreInst(StoreInst &I) {
2410 StoreList.push_back(Elt: &I);
2411 if (ClCheckAccessAddress)
2412 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2413 }
2414
2415 void handleCASOrRMW(Instruction &I) {
2416 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2417
2418 IRBuilder<> IRB(&I);
2419 Value *Addr = I.getOperand(i: 0);
2420 Value *Val = I.getOperand(i: 1);
2421 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, ShadowTy: getShadowTy(V: Val), Alignment: Align(1),
2422 /*isStore*/ true)
2423 .first;
2424
2425 if (ClCheckAccessAddress)
2426 insertCheckShadowOf(Val: Addr, OrigIns: &I);
2427
2428 // Only test the conditional argument of cmpxchg instruction.
2429 // The other argument can potentially be uninitialized, but we can not
2430 // detect this situation reliably without possible false positives.
2431 if (isa<AtomicCmpXchgInst>(Val: I))
2432 insertCheckShadowOf(Val, OrigIns: &I);
2433
2434 IRB.CreateStore(Val: getCleanShadow(V: Val), Ptr: ShadowPtr);
2435
2436 setShadow(V: &I, SV: getCleanShadow(V: &I));
2437 setOrigin(V: &I, Origin: getCleanOrigin());
2438 }
2439
2440 void visitAtomicRMWInst(AtomicRMWInst &I) {
2441 handleCASOrRMW(I);
2442 I.setOrdering(addReleaseOrdering(a: I.getOrdering()));
2443 }
2444
2445 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2446 handleCASOrRMW(I);
2447 I.setSuccessOrdering(addReleaseOrdering(a: I.getSuccessOrdering()));
2448 }
2449
2450 /// Generic handler to compute shadow for == and != comparisons.
2451 ///
2452 /// This function is used by handleEqualityComparison and visitSwitchInst.
2453 ///
2454 /// Sometimes the comparison result is known even if some of the bits of the
2455 /// arguments are not.
2456 Value *propagateEqualityComparison(IRBuilder<> &IRB, Value *A, Value *B,
2457 Value *Sa, Value *Sb) {
2458 assert(getShadowTy(A) == Sa->getType());
2459 assert(getShadowTy(B) == Sb->getType());
2460
2461 // Get rid of pointers and vectors of pointers.
2462 // For ints (and vectors of ints), types of A and Sa match,
2463 // and this is a no-op.
2464 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2465 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2466
2467 // A == B <==> (C = A^B) == 0
2468 // A != B <==> (C = A^B) != 0
2469 // Sc = Sa | Sb
2470 Value *C = IRB.CreateXor(LHS: A, RHS: B);
2471 Value *Sc = IRB.CreateOr(LHS: Sa, RHS: Sb);
2472 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2473 // Result is defined if one of the following is true
2474 // * there is a defined 1 bit in C
2475 // * C is fully defined
2476 // Si = !(C & ~Sc) && Sc
2477 Value *Zero = Constant::getNullValue(Ty: Sc->getType());
2478 Value *MinusOne = Constant::getAllOnesValue(Ty: Sc->getType());
2479 Value *LHS = IRB.CreateICmpNE(LHS: Sc, RHS: Zero);
2480 Value *RHS =
2481 IRB.CreateICmpEQ(LHS: IRB.CreateAnd(LHS: IRB.CreateXor(LHS: Sc, RHS: MinusOne), RHS: C), RHS: Zero);
2482 Value *Si = IRB.CreateAnd(LHS, RHS);
2483 Si->setName("_msprop_icmp");
2484
2485 return Si;
2486 }
2487
2488 // Instrument:
2489 // switch i32 %Val, label %else [ i32 0, label %A
2490 // i32 1, label %B
2491 // i32 2, label %C ]
2492 //
2493 // Typically, the switch input value (%Val) is fully initialized.
2494 //
2495 // Sometimes the compiler may convert (icmp + br) into a switch statement.
2496 // MSan allows icmp eq/ne with partly initialized inputs to still result in a
2497 // fully initialized output, if there exists a bit that is initialized in
2498 // both inputs with a differing value. For compatibility, we support this in
2499 // the switch instrumentation as well. Note that this edge case only applies
2500 // if the switch input value does not match *any* of the cases (matching any
2501 // of the cases requires an exact, fully initialized match).
2502 //
2503 // ShadowCases = 0
2504 // | propagateEqualityComparison(Val, 0)
2505 // | propagateEqualityComparison(Val, 1)
2506 // | propagateEqualityComparison(Val, 2))
2507 void visitSwitchInst(SwitchInst &SI) {
2508 IRBuilder<> IRB(&SI);
2509
2510 Value *Val = SI.getCondition();
2511 Value *ShadowVal = getShadow(V: Val);
2512 // TODO: add fast path - if the condition is fully initialized, we know
2513 // there is no UUM, without needing to consider the case values below.
2514
2515 // Some code (e.g., AMDGPUGenMCCodeEmitter.inc) has tens of thousands of
2516 // cases. This results in an extremely long chained expression for MSan's
2517 // switch instrumentation, which can cause the JumpThreadingPass to have a
2518 // stack overflow or excessive runtime. We limit the number of cases
2519 // considered, with the tradeoff of niche false negatives.
2520 // TODO: figure out a better solution.
2521 int casesToConsider = ClSwitchPrecision;
2522
2523 Value *ShadowCases = nullptr;
2524 for (auto Case : SI.cases()) {
2525 if (casesToConsider <= 0)
2526 break;
2527
2528 Value *Comparator = Case.getCaseValue();
2529 // TODO: some simplification is possible when comparing multiple cases
2530 // simultaneously.
2531 Value *ComparisonShadow = propagateEqualityComparison(
2532 IRB, A: Val, B: Comparator, Sa: ShadowVal, Sb: getShadow(V: Comparator));
2533
2534 if (ShadowCases)
2535 ShadowCases = IRB.CreateOr(LHS: ShadowCases, RHS: ComparisonShadow);
2536 else
2537 ShadowCases = ComparisonShadow;
2538
2539 casesToConsider--;
2540 }
2541
2542 if (ShadowCases)
2543 insertCheckShadow(Shadow: ShadowCases, Origin: getOrigin(V: Val), OrigIns: &SI);
2544 }
2545
2546 // Vector manipulation.
2547 void visitExtractElementInst(ExtractElementInst &I) {
2548 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 1), OrigIns: &I);
2549 IRBuilder<> IRB(&I);
2550 setShadow(V: &I, SV: IRB.CreateExtractElement(Vec: getShadow(I: &I, i: 0), Idx: I.getOperand(i_nocapture: 1),
2551 Name: "_msprop"));
2552 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2553 }
2554
2555 void visitInsertElementInst(InsertElementInst &I) {
2556 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 2), OrigIns: &I);
2557 IRBuilder<> IRB(&I);
2558 auto *Shadow0 = getShadow(I: &I, i: 0);
2559 auto *Shadow1 = getShadow(I: &I, i: 1);
2560 setShadow(V: &I, SV: IRB.CreateInsertElement(Vec: Shadow0, NewElt: Shadow1, Idx: I.getOperand(i_nocapture: 2),
2561 Name: "_msprop"));
2562 setOriginForNaryOp(I);
2563 }
2564
2565 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2566 IRBuilder<> IRB(&I);
2567 auto *Shadow0 = getShadow(I: &I, i: 0);
2568 auto *Shadow1 = getShadow(I: &I, i: 1);
2569 setShadow(V: &I, SV: IRB.CreateShuffleVector(V1: Shadow0, V2: Shadow1, Mask: I.getShuffleMask(),
2570 Name: "_msprop"));
2571 setOriginForNaryOp(I);
2572 }
2573
2574 // Casts.
2575 void visitSExtInst(SExtInst &I) {
2576 IRBuilder<> IRB(&I);
2577 setShadow(V: &I, SV: IRB.CreateSExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2578 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2579 }
2580
2581 void visitZExtInst(ZExtInst &I) {
2582 IRBuilder<> IRB(&I);
2583 setShadow(V: &I, SV: IRB.CreateZExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2584 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2585 }
2586
2587 void visitTruncInst(TruncInst &I) {
2588 IRBuilder<> IRB(&I);
2589 setShadow(V: &I, SV: IRB.CreateTrunc(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2590 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2591 }
2592
2593 void visitBitCastInst(BitCastInst &I) {
2594 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2595 // a musttail call and a ret, don't instrument. New instructions are not
2596 // allowed after a musttail call.
2597 if (auto *CI = dyn_cast<CallInst>(Val: I.getOperand(i_nocapture: 0)))
2598 if (CI->isMustTailCall())
2599 return;
2600 IRBuilder<> IRB(&I);
2601 setShadow(V: &I, SV: IRB.CreateBitCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I)));
2602 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2603 }
2604
2605 void visitPtrToIntInst(PtrToIntInst &I) {
2606 IRBuilder<> IRB(&I);
2607 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2608 Name: "_msprop_ptrtoint"));
2609 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2610 }
2611
2612 void visitPtrToAddrInst(PtrToAddrInst &I) {
2613 IRBuilder<> IRB(&I);
2614 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2615 Name: "_msprop_ptrtoaddr"));
2616 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2617 }
2618
2619 void visitIntToPtrInst(IntToPtrInst &I) {
2620 IRBuilder<> IRB(&I);
2621 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2622 Name: "_msprop_inttoptr"));
2623 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2624 }
2625
2626 /// Handle LLVM and NEON vector convert intrinsics.
2627 ///
2628 /// e.g., <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>)
2629 /// i32 @llvm.aarch64.neon.fcvtms.i32.f64 (double)
2630 /// <2 x i32> @fptoui (<2 x float>)
2631 /// i64 @llvm.fptosi.sat.i64.f64(double)
2632 ///
2633 /// Note that the size of input/output elements can differ e.g.,
2634 /// double @sitofp(i32)
2635 /// but the number of elements must be the same.
2636 ///
2637 /// For conversions to or from fixed-point, there is a trailing argument to
2638 /// indicate the fixed-point precision:
2639 /// - <4 x float> llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32>, i32)
2640 /// - <4 x i32> llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float>, i32)
2641 ///
2642 /// For x86 SSE vector convert intrinsics, see
2643 /// handleSSEVectorConvertIntrinsic().
2644 void handleGenericVectorConvertIntrinsic(Instruction &I, bool FixedPoint) {
2645 [[maybe_unused]] unsigned NumArgs = I.getNumOperands();
2646 if (auto *CI = dyn_cast<CallInst>(Val: &I))
2647 NumArgs = CI->arg_size();
2648
2649 if (FixedPoint) {
2650 assert(NumArgs == 2);
2651 Value *Precision = I.getOperand(i: 1);
2652 insertCheckShadowOf(Val: Precision, OrigIns: &I);
2653 } else {
2654 assert(NumArgs == 1);
2655 }
2656
2657 IRBuilder<> IRB(&I);
2658 Value *S0 = getShadow(I: &I, i: 0);
2659
2660 /// For scalars:
2661 /// Since they are converting from floating-point to integer, or between
2662 /// different width floating-point values, the output is:
2663 /// - fully uninitialized if *any* bit of the input is uninitialized
2664 /// - fully ininitialized if all bits of the input are ininitialized
2665 /// We apply the same principle on a per-field basis for vectors.
2666 Value *OutShadow = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)),
2667 DestTy: getShadowTy(V: &I));
2668 setShadow(V: &I, SV: OutShadow);
2669 setOriginForNaryOp(I);
2670 }
2671
2672 void visitFPToSIInst(CastInst &I) {
2673 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2674 }
2675 void visitFPToUIInst(CastInst &I) {
2676 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2677 }
2678 void visitSIToFPInst(CastInst &I) {
2679 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2680 }
2681 void visitUIToFPInst(CastInst &I) {
2682 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2683 }
2684
2685 void visitFPExtInst(CastInst &I) {
2686 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2687 }
2688 void visitFPTruncInst(CastInst &I) {
2689 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2690 }
2691
2692 /// Generic handler to compute shadow for bitwise AND.
2693 ///
2694 /// This is used by 'visitAnd' but also as a primitive for other handlers.
2695 ///
2696 /// This code is precise: it implements the rule that "And" of an initialized
2697 /// zero bit always results in an initialized value:
2698 // 1&1 => 1; 0&1 => 0; p&1 => p;
2699 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2700 // 1&p => p; 0&p => 0; p&p => p;
2701 //
2702 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2703 Value *handleBitwiseAnd(IRBuilder<> &IRB, Value *V1, Value *V2, Value *S1,
2704 Value *S2) {
2705 // "The two arguments to the ‘and’ instruction must be integer or vector
2706 // of integer values. Both arguments must have identical types."
2707 //
2708 // We enforce this condition for all callers to handleBitwiseAnd(); callers
2709 // with non-integer types should call CreateAppToShadowCast() themselves.
2710 assert(V1->getType()->isIntOrIntVectorTy());
2711 assert(V1->getType() == V2->getType());
2712
2713 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2714 assert(V1->getType() == S1->getType());
2715 assert(V2->getType() == S2->getType());
2716
2717 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2718 Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2719 Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2720
2721 return IRB.CreateOr(Ops: {S1S2, V1S2, S1V2});
2722 }
2723
2724 /// Handler for bitwise AND operator.
2725 void visitAnd(BinaryOperator &I) {
2726 IRBuilder<> IRB(&I);
2727 Value *V1 = I.getOperand(i_nocapture: 0);
2728 Value *V2 = I.getOperand(i_nocapture: 1);
2729 Value *S1 = getShadow(I: &I, i: 0);
2730 Value *S2 = getShadow(I: &I, i: 1);
2731
2732 Value *OutShadow = handleBitwiseAnd(IRB, V1, V2, S1, S2);
2733
2734 setShadow(V: &I, SV: OutShadow);
2735 setOriginForNaryOp(I);
2736 }
2737
2738 void visitOr(BinaryOperator &I) {
2739 IRBuilder<> IRB(&I);
2740 // "Or" of 1 and a poisoned value results in unpoisoned value:
2741 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2742 // 1|0 => 1; 0|0 => 0; p|0 => p;
2743 // 1|p => 1; 0|p => p; p|p => p;
2744 //
2745 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2746 //
2747 // If the "disjoint OR" property is violated, the result is poison, and
2748 // hence the entire shadow is uninitialized:
2749 // S = S | SignExt(V1 & V2 != 0)
2750 Value *S1 = getShadow(I: &I, i: 0);
2751 Value *S2 = getShadow(I: &I, i: 1);
2752 Value *V1 = I.getOperand(i_nocapture: 0);
2753 Value *V2 = I.getOperand(i_nocapture: 1);
2754
2755 // "The two arguments to the ‘or’ instruction must be integer or vector
2756 // of integer values. Both arguments must have identical types."
2757 assert(V1->getType()->isIntOrIntVectorTy());
2758 assert(V1->getType() == V2->getType());
2759
2760 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2761 assert(V1->getType() == S1->getType());
2762 assert(V2->getType() == S2->getType());
2763
2764 Value *NotV1 = IRB.CreateNot(V: V1);
2765 Value *NotV2 = IRB.CreateNot(V: V2);
2766
2767 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2768 Value *S2NotV1 = IRB.CreateAnd(LHS: NotV1, RHS: S2);
2769 Value *S1NotV2 = IRB.CreateAnd(LHS: S1, RHS: NotV2);
2770
2771 Value *S = IRB.CreateOr(Ops: {S1S2, S2NotV1, S1NotV2});
2772
2773 if (ClPreciseDisjointOr && cast<PossiblyDisjointInst>(Val: &I)->isDisjoint()) {
2774 Value *V1V2 = IRB.CreateAnd(LHS: V1, RHS: V2);
2775 Value *DisjointOrShadow = IRB.CreateSExt(
2776 V: IRB.CreateICmpNE(LHS: V1V2, RHS: getCleanShadow(V: V1V2)), DestTy: V1V2->getType());
2777 S = IRB.CreateOr(LHS: S, RHS: DisjointOrShadow, Name: "_ms_disjoint");
2778 }
2779
2780 setShadow(V: &I, SV: S);
2781 setOriginForNaryOp(I);
2782 }
2783
2784 /// Default propagation of shadow and/or origin.
2785 ///
2786 /// This class implements the general case of shadow propagation, used in all
2787 /// cases where we don't know and/or don't care about what the operation
2788 /// actually does. It converts all input shadow values to a common type
2789 /// (extending or truncating as necessary), and bitwise OR's them.
2790 ///
2791 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2792 /// fully initialized), and less prone to false positives.
2793 ///
2794 /// This class also implements the general case of origin propagation. For a
2795 /// Nary operation, result origin is set to the origin of an argument that is
2796 /// not entirely initialized. If there is more than one such arguments, the
2797 /// rightmost of them is picked. It does not matter which one is picked if all
2798 /// arguments are initialized.
2799 template <bool CombineShadow> class Combiner {
2800 Value *Shadow = nullptr;
2801 Value *Origin = nullptr;
2802 IRBuilder<> &IRB;
2803 MemorySanitizerVisitor *MSV;
2804
2805 public:
2806 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2807 : IRB(IRB), MSV(MSV) {}
2808
2809 /// Add a pair of shadow and origin values to the mix.
2810 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2811 if (CombineShadow) {
2812 assert(OpShadow);
2813 if (!Shadow)
2814 Shadow = OpShadow;
2815 else {
2816 OpShadow = MSV->CreateShadowCast(IRB, V: OpShadow, dstTy: Shadow->getType());
2817 Shadow = IRB.CreateOr(LHS: Shadow, RHS: OpShadow, Name: "_msprop");
2818 }
2819 }
2820
2821 if (MSV->MS.TrackOrigins) {
2822 assert(OpOrigin);
2823 if (!Origin) {
2824 Origin = OpOrigin;
2825 } else {
2826 Constant *ConstOrigin = dyn_cast<Constant>(Val: OpOrigin);
2827 // No point in adding something that might result in 0 origin value.
2828 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2829 Value *Cond = MSV->convertToBool(V: OpShadow, IRB);
2830 Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2831 }
2832 }
2833 }
2834 return *this;
2835 }
2836
2837 /// Add an application value to the mix.
2838 Combiner &Add(Value *V) {
2839 Value *OpShadow = MSV->getShadow(V);
2840 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2841 return Add(OpShadow, OpOrigin);
2842 }
2843
2844 /// Set the current combined values as the given instruction's shadow
2845 /// and origin.
2846 void Done(Instruction *I) {
2847 if (CombineShadow) {
2848 assert(Shadow);
2849 Shadow = MSV->CreateShadowCast(IRB, V: Shadow, dstTy: MSV->getShadowTy(V: I));
2850 MSV->setShadow(V: I, SV: Shadow);
2851 }
2852 if (MSV->MS.TrackOrigins) {
2853 assert(Origin);
2854 MSV->setOrigin(V: I, Origin);
2855 }
2856 }
2857
2858 /// Store the current combined value at the specified origin
2859 /// location.
2860 void DoneAndStoreOrigin(TypeSize TS, Value *OriginPtr) {
2861 if (MSV->MS.TrackOrigins) {
2862 assert(Origin);
2863 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, Alignment: kMinOriginAlignment);
2864 }
2865 }
2866 };
2867
2868 using ShadowAndOriginCombiner = Combiner<true>;
2869 using OriginCombiner = Combiner<false>;
2870
2871 /// Propagate origin for arbitrary operation.
2872 void setOriginForNaryOp(Instruction &I) {
2873 if (!MS.TrackOrigins)
2874 return;
2875 IRBuilder<> IRB(&I);
2876 OriginCombiner OC(this, IRB);
2877 for (Use &Op : I.operands())
2878 OC.Add(V: Op.get());
2879 OC.Done(I: &I);
2880 }
2881
2882 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2883 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2884 "Vector of pointers is not a valid shadow type");
2885 return Ty->isVectorTy() ? cast<FixedVectorType>(Val: Ty)->getNumElements() *
2886 Ty->getScalarSizeInBits()
2887 : Ty->getPrimitiveSizeInBits();
2888 }
2889
2890 /// Cast between two shadow types, extending or truncating as
2891 /// necessary.
2892 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2893 bool Signed = false) {
2894 Type *srcTy = V->getType();
2895 if (srcTy == dstTy)
2896 return V;
2897 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: srcTy);
2898 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: dstTy);
2899 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2900 return IRB.CreateICmpNE(LHS: V, RHS: getCleanShadow(V));
2901
2902 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2903 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2904 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2905 cast<VectorType>(Val: dstTy)->getElementCount() ==
2906 cast<VectorType>(Val: srcTy)->getElementCount())
2907 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2908 Value *V1 = IRB.CreateBitCast(V, DestTy: Type::getIntNTy(C&: *MS.C, N: srcSizeInBits));
2909 Value *V2 =
2910 IRB.CreateIntCast(V: V1, DestTy: Type::getIntNTy(C&: *MS.C, N: dstSizeInBits), isSigned: Signed);
2911 return IRB.CreateBitCast(V: V2, DestTy: dstTy);
2912 // TODO: handle struct types.
2913 }
2914
2915 /// Cast an application value to the type of its own shadow.
2916 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2917 Type *ShadowTy = getShadowTy(V);
2918 if (V->getType() == ShadowTy)
2919 return V;
2920 if (V->getType()->isPtrOrPtrVectorTy())
2921 return IRB.CreatePtrToInt(V, DestTy: ShadowTy);
2922 else
2923 return IRB.CreateBitCast(V, DestTy: ShadowTy);
2924 }
2925
2926 /// Propagate shadow for arbitrary operation.
2927 void handleShadowOr(Instruction &I) {
2928 IRBuilder<> IRB(&I);
2929 ShadowAndOriginCombiner SC(this, IRB);
2930 for (Use &Op : I.operands())
2931 SC.Add(V: Op.get());
2932 SC.Done(I: &I);
2933 }
2934
2935 // Perform a bitwise OR on the horizontal pairs (or other specified grouping)
2936 // of elements.
2937 //
2938 // For example, suppose we have:
2939 // VectorA: <a0, a1, a2, a3, a4, a5>
2940 // VectorB: <b0, b1, b2, b3, b4, b5>
2941 // ReductionFactor: 3
2942 // Shards: 1
2943 // The output would be:
2944 // <a0|a1|a2, a3|a4|a5, b0|b1|b2, b3|b4|b5>
2945 //
2946 // If we have:
2947 // VectorA: <a0, a1, a2, a3, a4, a5, a6, a7>
2948 // VectorB: <b0, b1, b2, b3, b4, b5, b6, b7>
2949 // ReductionFactor: 2
2950 // Shards: 2
2951 // then a and be each have 2 "shards", resulting in the output being
2952 // interleaved:
2953 // <a0|a1, a2|a3, b0|b1, b2|b3, a4|a5, a6|a7, b4|b5, b6|b7>
2954 //
2955 // This is convenient for instrumenting horizontal add/sub.
2956 // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic().
2957 Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor,
2958 unsigned Shards, Value *VectorA, Value *VectorB) {
2959 assert(isa<FixedVectorType>(VectorA->getType()));
2960 unsigned NumElems =
2961 cast<FixedVectorType>(Val: VectorA->getType())->getNumElements();
2962
2963 [[maybe_unused]] unsigned TotalNumElems = NumElems;
2964 if (VectorB) {
2965 assert(VectorA->getType() == VectorB->getType());
2966 TotalNumElems *= 2;
2967 }
2968
2969 assert(NumElems % (ReductionFactor * Shards) == 0);
2970
2971 Value *Or = nullptr;
2972
2973 IRBuilder<> IRB(&I);
2974 for (unsigned i = 0; i < ReductionFactor; i++) {
2975 SmallVector<int, 16> Mask;
2976
2977 for (unsigned j = 0; j < Shards; j++) {
2978 unsigned Offset = NumElems / Shards * j;
2979
2980 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2981 Mask.push_back(Elt: Offset + X + i);
2982
2983 if (VectorB) {
2984 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2985 Mask.push_back(Elt: NumElems + Offset + X + i);
2986 }
2987 }
2988
2989 Value *Masked;
2990 if (VectorB)
2991 Masked = IRB.CreateShuffleVector(V1: VectorA, V2: VectorB, Mask);
2992 else
2993 Masked = IRB.CreateShuffleVector(V: VectorA, Mask);
2994
2995 if (Or)
2996 Or = IRB.CreateOr(LHS: Or, RHS: Masked);
2997 else
2998 Or = Masked;
2999 }
3000
3001 return Or;
3002 }
3003
3004 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
3005 /// fields.
3006 ///
3007 /// e.g., <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16>)
3008 /// <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8>, <16 x i8>)
3009 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards) {
3010 assert(I.arg_size() == 1 || I.arg_size() == 2);
3011
3012 assert(I.getType()->isVectorTy());
3013 assert(I.getArgOperand(0)->getType()->isVectorTy());
3014
3015 [[maybe_unused]] FixedVectorType *ParamType =
3016 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
3017 assert((I.arg_size() != 2) ||
3018 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
3019 [[maybe_unused]] FixedVectorType *ReturnType =
3020 cast<FixedVectorType>(Val: I.getType());
3021 assert(ParamType->getNumElements() * I.arg_size() ==
3022 2 * ReturnType->getNumElements());
3023
3024 IRBuilder<> IRB(&I);
3025
3026 // Horizontal OR of shadow
3027 Value *FirstArgShadow = getShadow(I: &I, i: 0);
3028 Value *SecondArgShadow = nullptr;
3029 if (I.arg_size() == 2)
3030 SecondArgShadow = getShadow(I: &I, i: 1);
3031
3032 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
3033 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
3034
3035 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
3036
3037 setShadow(V: &I, SV: OrShadow);
3038 setOriginForNaryOp(I);
3039 }
3040
3041 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
3042 /// fields, with the parameters reinterpreted to have elements of a specified
3043 /// width. For example:
3044 /// @llvm.x86.ssse3.phadd.w(<1 x i64> [[VAR1]], <1 x i64> [[VAR2]])
3045 /// conceptually operates on
3046 /// (<4 x i16> [[VAR1]], <4 x i16> [[VAR2]])
3047 /// and can be handled with ReinterpretElemWidth == 16.
3048 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards,
3049 int ReinterpretElemWidth) {
3050 assert(I.arg_size() == 1 || I.arg_size() == 2);
3051
3052 assert(I.getType()->isVectorTy());
3053 assert(I.getArgOperand(0)->getType()->isVectorTy());
3054
3055 FixedVectorType *ParamType =
3056 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
3057 assert((I.arg_size() != 2) ||
3058 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
3059
3060 [[maybe_unused]] FixedVectorType *ReturnType =
3061 cast<FixedVectorType>(Val: I.getType());
3062 assert(ParamType->getNumElements() * I.arg_size() ==
3063 2 * ReturnType->getNumElements());
3064
3065 IRBuilder<> IRB(&I);
3066
3067 FixedVectorType *ReinterpretShadowTy = nullptr;
3068 assert(isAligned(Align(ReinterpretElemWidth),
3069 ParamType->getPrimitiveSizeInBits()));
3070 ReinterpretShadowTy = FixedVectorType::get(
3071 ElementType: IRB.getIntNTy(N: ReinterpretElemWidth),
3072 NumElts: ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth);
3073
3074 // Horizontal OR of shadow
3075 Value *FirstArgShadow = getShadow(I: &I, i: 0);
3076 FirstArgShadow = IRB.CreateBitCast(V: FirstArgShadow, DestTy: ReinterpretShadowTy);
3077
3078 // If we had two parameters each with an odd number of elements, the total
3079 // number of elements is even, but we have never seen this in extant
3080 // instruction sets, so we enforce that each parameter must have an even
3081 // number of elements.
3082 assert(isAligned(
3083 Align(2),
3084 cast<FixedVectorType>(FirstArgShadow->getType())->getNumElements()));
3085
3086 Value *SecondArgShadow = nullptr;
3087 if (I.arg_size() == 2) {
3088 SecondArgShadow = getShadow(I: &I, i: 1);
3089 SecondArgShadow = IRB.CreateBitCast(V: SecondArgShadow, DestTy: ReinterpretShadowTy);
3090 }
3091
3092 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
3093 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
3094
3095 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
3096
3097 setShadow(V: &I, SV: OrShadow);
3098 setOriginForNaryOp(I);
3099 }
3100
3101 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
3102
3103 // Handle multiplication by constant.
3104 //
3105 // Handle a special case of multiplication by constant that may have one or
3106 // more zeros in the lower bits. This makes corresponding number of lower bits
3107 // of the result zero as well. We model it by shifting the other operand
3108 // shadow left by the required number of bits. Effectively, we transform
3109 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
3110 // We use multiplication by 2**N instead of shift to cover the case of
3111 // multiplication by 0, which may occur in some elements of a vector operand.
3112 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
3113 Value *OtherArg) {
3114 Constant *ShadowMul;
3115 Type *Ty = ConstArg->getType();
3116 if (auto *VTy = dyn_cast<VectorType>(Val: Ty)) {
3117 unsigned NumElements = cast<FixedVectorType>(Val: VTy)->getNumElements();
3118 Type *EltTy = VTy->getElementType();
3119 SmallVector<Constant *, 16> Elements;
3120 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
3121 if (ConstantInt *Elt =
3122 dyn_cast<ConstantInt>(Val: ConstArg->getAggregateElement(Elt: Idx))) {
3123 const APInt &V = Elt->getValue();
3124 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
3125 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: V2));
3126 } else {
3127 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: 1));
3128 }
3129 }
3130 ShadowMul = ConstantVector::get(V: Elements);
3131 } else {
3132 if (ConstantInt *Elt = dyn_cast<ConstantInt>(Val: ConstArg)) {
3133 const APInt &V = Elt->getValue();
3134 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
3135 ShadowMul = ConstantInt::get(Ty, V: V2);
3136 } else {
3137 ShadowMul = ConstantInt::get(Ty, V: 1);
3138 }
3139 }
3140
3141 IRBuilder<> IRB(&I);
3142 setShadow(V: &I,
3143 SV: IRB.CreateMul(LHS: getShadow(V: OtherArg), RHS: ShadowMul, Name: "msprop_mul_cst"));
3144 setOrigin(V: &I, Origin: getOrigin(V: OtherArg));
3145 }
3146
3147 void visitMul(BinaryOperator &I) {
3148 Constant *constOp0 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0));
3149 Constant *constOp1 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3150 if (constOp0 && !constOp1)
3151 handleMulByConstant(I, ConstArg: constOp0, OtherArg: I.getOperand(i_nocapture: 1));
3152 else if (constOp1 && !constOp0)
3153 handleMulByConstant(I, ConstArg: constOp1, OtherArg: I.getOperand(i_nocapture: 0));
3154 else
3155 handleShadowOr(I);
3156 }
3157
3158 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
3159 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
3160 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
3161 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
3162 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
3163 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
3164
3165 void handleIntegerDiv(Instruction &I) {
3166 IRBuilder<> IRB(&I);
3167 // Strict on the second argument.
3168 insertCheckShadowOf(Val: I.getOperand(i: 1), OrigIns: &I);
3169 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3170 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3171 }
3172
3173 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3174 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3175 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
3176 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
3177
3178 // Floating point division is side-effect free. We can not require that the
3179 // divisor is fully initialized and must propagate shadow. See PR37523.
3180 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
3181 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
3182
3183 /// Instrument == and != comparisons.
3184 ///
3185 /// Sometimes the comparison result is known even if some of the bits of the
3186 /// arguments are not.
3187 void handleEqualityComparison(ICmpInst &I) {
3188 IRBuilder<> IRB(&I);
3189 Value *A = I.getOperand(i_nocapture: 0);
3190 Value *B = I.getOperand(i_nocapture: 1);
3191 Value *Sa = getShadow(V: A);
3192 Value *Sb = getShadow(V: B);
3193
3194 Value *Si = propagateEqualityComparison(IRB, A, B, Sa, Sb);
3195
3196 setShadow(V: &I, SV: Si);
3197 setOriginForNaryOp(I);
3198 }
3199
3200 /// Instrument relational comparisons.
3201 ///
3202 /// This function does exact shadow propagation for all relational
3203 /// comparisons of integers, pointers and vectors of those.
3204 /// FIXME: output seems suboptimal when one of the operands is a constant
3205 void handleRelationalComparisonExact(ICmpInst &I) {
3206 IRBuilder<> IRB(&I);
3207 Value *A = I.getOperand(i_nocapture: 0);
3208 Value *B = I.getOperand(i_nocapture: 1);
3209 Value *Sa = getShadow(V: A);
3210 Value *Sb = getShadow(V: B);
3211
3212 // Get rid of pointers and vectors of pointers.
3213 // For ints (and vectors of ints), types of A and Sa match,
3214 // and this is a no-op.
3215 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
3216 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
3217
3218 // Let [a0, a1] be the interval of possible values of A, taking into account
3219 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
3220 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
3221 bool IsSigned = I.isSigned();
3222
3223 auto GetMinMaxUnsigned = [&](Value *V, Value *S) {
3224 if (IsSigned) {
3225 // Sign-flip to map from signed range to unsigned range. Relation A vs B
3226 // should be preserved, if checked with `getUnsignedPredicate()`.
3227 // Relationship between Amin, Amax, Bmin, Bmax also will not be
3228 // affected, as they are created by effectively adding/substructing from
3229 // A (or B) a value, derived from shadow, with no overflow, either
3230 // before or after sign flip.
3231 APInt MinVal =
3232 APInt::getSignedMinValue(numBits: V->getType()->getScalarSizeInBits());
3233 V = IRB.CreateXor(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: MinVal));
3234 }
3235 // Minimize undefined bits.
3236 Value *Min = IRB.CreateAnd(LHS: V, RHS: IRB.CreateNot(V: S));
3237 Value *Max = IRB.CreateOr(LHS: V, RHS: S);
3238 return std::make_pair(x&: Min, y&: Max);
3239 };
3240
3241 auto [Amin, Amax] = GetMinMaxUnsigned(A, Sa);
3242 auto [Bmin, Bmax] = GetMinMaxUnsigned(B, Sb);
3243 Value *S1 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amin, RHS: Bmax);
3244 Value *S2 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amax, RHS: Bmin);
3245
3246 Value *Si = IRB.CreateXor(LHS: S1, RHS: S2);
3247 setShadow(V: &I, SV: Si);
3248 setOriginForNaryOp(I);
3249 }
3250
3251 /// Instrument signed relational comparisons.
3252 ///
3253 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
3254 /// bit of the shadow. Everything else is delegated to handleShadowOr().
3255 void handleSignedRelationalComparison(ICmpInst &I) {
3256 Constant *constOp;
3257 Value *op = nullptr;
3258 CmpInst::Predicate pre;
3259 if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3260 op = I.getOperand(i_nocapture: 0);
3261 pre = I.getPredicate();
3262 } else if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0)))) {
3263 op = I.getOperand(i_nocapture: 1);
3264 pre = I.getSwappedPredicate();
3265 } else {
3266 handleShadowOr(I);
3267 return;
3268 }
3269
3270 if ((constOp->isNullValue() &&
3271 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
3272 (constOp->isAllOnesValue() &&
3273 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
3274 IRBuilder<> IRB(&I);
3275 Value *Shadow = IRB.CreateICmpSLT(LHS: getShadow(V: op), RHS: getCleanShadow(V: op),
3276 Name: "_msprop_icmp_s");
3277 setShadow(V: &I, SV: Shadow);
3278 setOrigin(V: &I, Origin: getOrigin(V: op));
3279 } else {
3280 handleShadowOr(I);
3281 }
3282 }
3283
3284 void visitICmpInst(ICmpInst &I) {
3285 if (!ClHandleICmp) {
3286 handleShadowOr(I);
3287 return;
3288 }
3289 if (I.isEquality()) {
3290 handleEqualityComparison(I);
3291 return;
3292 }
3293
3294 assert(I.isRelational());
3295 if (ClHandleICmpExact) {
3296 handleRelationalComparisonExact(I);
3297 return;
3298 }
3299 if (I.isSigned()) {
3300 handleSignedRelationalComparison(I);
3301 return;
3302 }
3303
3304 assert(I.isUnsigned());
3305 if ((isa<Constant>(Val: I.getOperand(i_nocapture: 0)) || isa<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3306 handleRelationalComparisonExact(I);
3307 return;
3308 }
3309
3310 handleShadowOr(I);
3311 }
3312
3313 void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
3314
3315 void handleShift(BinaryOperator &I) {
3316 IRBuilder<> IRB(&I);
3317 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3318 // Otherwise perform the same shift on S1.
3319 Value *S1 = getShadow(I: &I, i: 0);
3320 Value *S2 = getShadow(I: &I, i: 1);
3321 Value *S2Conv =
3322 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3323 Value *V2 = I.getOperand(i_nocapture: 1);
3324 Value *Shift = IRB.CreateBinOp(Opc: I.getOpcode(), LHS: S1, RHS: V2);
3325 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3326 setOriginForNaryOp(I);
3327 }
3328
3329 void visitShl(BinaryOperator &I) { handleShift(I); }
3330 void visitAShr(BinaryOperator &I) { handleShift(I); }
3331 void visitLShr(BinaryOperator &I) { handleShift(I); }
3332
3333 void handleFunnelShift(IntrinsicInst &I) {
3334 IRBuilder<> IRB(&I);
3335 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3336 // Otherwise perform the same shift on S0 and S1.
3337 Value *S0 = getShadow(I: &I, i: 0);
3338 Value *S1 = getShadow(I: &I, i: 1);
3339 Value *S2 = getShadow(I: &I, i: 2);
3340 Value *S2Conv =
3341 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3342 Value *V2 = I.getOperand(i_nocapture: 2);
3343 Value *Shift = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: S2Conv->getType(),
3344 Args: {S0, S1, V2});
3345 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3346 setOriginForNaryOp(I);
3347 }
3348
3349 // Instrument bit manipulation intrinsics.
3350 // All of these intrinsics are Z = I(SRC, MASK)
3351 // where the types of all operands and the result match.
3352 // The following instrumentation happens to work for all of them:
3353 // Sz = I(Ssrc, MASK) | (sext (Smask != 0))
3354 void handleGenericBitManipulation(IntrinsicInst &I) {
3355 IRBuilder<> IRB(&I);
3356 Type *ShadowTy = getShadowTy(V: &I);
3357
3358 // If any bit of the mask operand is poisoned, then the whole thing is.
3359 Value *SMask = getShadow(I: &I, i: 1);
3360 SMask = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SMask, RHS: getCleanShadow(OrigTy: ShadowTy)),
3361 DestTy: ShadowTy);
3362 // Apply the same intrinsic to the shadow of the first operand.
3363 Value *S;
3364 if (Function *Func = I.getCalledFunction())
3365 S = IRB.CreateCall(Callee: Func, Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3366 else
3367 S = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: ShadowTy,
3368 Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3369
3370 setShadow(V: &I, SV: IRB.CreateOr(LHS: SMask, RHS: S));
3371 setOriginForNaryOp(I);
3372 }
3373
3374 /// Instrument llvm.memmove
3375 ///
3376 /// At this point we don't know if llvm.memmove will be inlined or not.
3377 /// If we don't instrument it and it gets inlined,
3378 /// our interceptor will not kick in and we will lose the memmove.
3379 /// If we instrument the call here, but it does not get inlined,
3380 /// we will memmove the shadow twice: which is bad in case
3381 /// of overlapping regions. So, we simply lower the intrinsic to a call.
3382 ///
3383 /// Similar situation exists for memcpy and memset.
3384 void visitMemMoveInst(MemMoveInst &I) {
3385 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3386 IRBuilder<> IRB(&I);
3387 IRB.CreateCall(Callee: MS.MemmoveFn,
3388 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3389 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3390 I.eraseFromParent();
3391 }
3392
3393 /// Instrument memcpy
3394 ///
3395 /// Similar to memmove: avoid copying shadow twice. This is somewhat
3396 /// unfortunate as it may slowdown small constant memcpys.
3397 /// FIXME: consider doing manual inline for small constant sizes and proper
3398 /// alignment.
3399 ///
3400 /// Note: This also handles memcpy.inline, which promises no calls to external
3401 /// functions as an optimization. However, with instrumentation enabled this
3402 /// is difficult to promise; additionally, we know that the MSan runtime
3403 /// exists and provides __msan_memcpy(). Therefore, we assume that with
3404 /// instrumentation it's safe to turn memcpy.inline into a call to
3405 /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
3406 /// itself, instrumentation should be disabled with the no_sanitize attribute.
3407 void visitMemCpyInst(MemCpyInst &I) {
3408 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3409 IRBuilder<> IRB(&I);
3410 IRB.CreateCall(Callee: MS.MemcpyFn,
3411 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3412 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3413 I.eraseFromParent();
3414 }
3415
3416 // Same as memcpy.
3417 void visitMemSetInst(MemSetInst &I) {
3418 IRBuilder<> IRB(&I);
3419 IRB.CreateCall(
3420 Callee: MS.MemsetFn,
3421 Args: {I.getArgOperand(i: 0),
3422 IRB.CreateIntCast(V: I.getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
3423 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3424 I.eraseFromParent();
3425 }
3426
3427 void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
3428
3429 void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
3430
3431 /// Handle vector store-like intrinsics.
3432 ///
3433 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
3434 /// has 1 pointer argument and 1 vector argument, returns void.
3435 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
3436 assert(I.arg_size() == 2);
3437
3438 IRBuilder<> IRB(&I);
3439 Value *Addr = I.getArgOperand(i: 0);
3440 Value *Shadow = getShadow(I: &I, i: 1);
3441 Value *ShadowPtr, *OriginPtr;
3442
3443 // We don't know the pointer alignment (could be unaligned SSE store!).
3444 // Have to assume to worst case.
3445 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
3446 Addr, IRB, ShadowTy: Shadow->getType(), Alignment: Align(1), /*isStore*/ true);
3447 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1));
3448
3449 if (ClCheckAccessAddress)
3450 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3451
3452 // FIXME: factor out common code from materializeStores
3453 if (MS.TrackOrigins)
3454 IRB.CreateStore(Val: getOrigin(I: &I, i: 1), Ptr: OriginPtr);
3455 return true;
3456 }
3457
3458 /// Handle vector load-like intrinsics.
3459 ///
3460 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
3461 /// has 1 pointer argument, returns a vector.
3462 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
3463 assert(I.arg_size() == 1);
3464
3465 IRBuilder<> IRB(&I);
3466 Value *Addr = I.getArgOperand(i: 0);
3467
3468 Type *ShadowTy = getShadowTy(V: &I);
3469 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
3470 if (PropagateShadow) {
3471 // We don't know the pointer alignment (could be unaligned SSE load!).
3472 // Have to assume to worst case.
3473 const Align Alignment = Align(1);
3474 std::tie(args&: ShadowPtr, args&: OriginPtr) =
3475 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3476 setShadow(V: &I,
3477 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
3478 } else {
3479 setShadow(V: &I, SV: getCleanShadow(V: &I));
3480 }
3481
3482 if (ClCheckAccessAddress)
3483 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3484
3485 if (MS.TrackOrigins) {
3486 if (PropagateShadow)
3487 setOrigin(V: &I, Origin: IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
3488 else
3489 setOrigin(V: &I, Origin: getCleanOrigin());
3490 }
3491 return true;
3492 }
3493
3494 /// Handle (SIMD arithmetic)-like intrinsics.
3495 ///
3496 /// Instrument intrinsics with any number of arguments of the same type [*],
3497 /// equal to the return type, plus a specified number of trailing flags of
3498 /// any type.
3499 ///
3500 /// [*] The type should be simple (no aggregates or pointers; vectors are
3501 /// fine).
3502 ///
3503 /// Caller guarantees that this intrinsic does not access memory.
3504 ///
3505 /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
3506 /// by this handler. See horizontalReduce().
3507 ///
3508 /// TODO: permutation intrinsics are also often incorrectly matched.
3509 [[maybe_unused]] bool
3510 maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
3511 unsigned int trailingFlags) {
3512 Type *RetTy = I.getType();
3513 if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy()))
3514 return false;
3515
3516 unsigned NumArgOperands = I.arg_size();
3517 assert(NumArgOperands >= trailingFlags);
3518 for (unsigned i = 0; i < NumArgOperands - trailingFlags; ++i) {
3519 Type *Ty = I.getArgOperand(i)->getType();
3520 if (Ty != RetTy)
3521 return false;
3522 }
3523
3524 IRBuilder<> IRB(&I);
3525 ShadowAndOriginCombiner SC(this, IRB);
3526 for (unsigned i = 0; i < NumArgOperands; ++i)
3527 SC.Add(V: I.getArgOperand(i));
3528 SC.Done(I: &I);
3529
3530 return true;
3531 }
3532
3533 /// Returns whether it was able to heuristically instrument unknown
3534 /// intrinsics.
3535 ///
3536 /// The main purpose of this code is to do something reasonable with all
3537 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
3538 /// We recognize several classes of intrinsics by their argument types and
3539 /// ModRefBehaviour and apply special instrumentation when we are reasonably
3540 /// sure that we know what the intrinsic does.
3541 ///
3542 /// We special-case intrinsics where this approach fails. See llvm.bswap
3543 /// handling as an example of that.
3544 bool maybeHandleUnknownIntrinsicUnlogged(IntrinsicInst &I) {
3545 unsigned NumArgOperands = I.arg_size();
3546 if (NumArgOperands == 0)
3547 return false;
3548
3549 if (NumArgOperands == 2 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3550 I.getArgOperand(i: 1)->getType()->isVectorTy() &&
3551 I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
3552 // This looks like a vector store.
3553 return handleVectorStoreIntrinsic(I);
3554 }
3555
3556 if (NumArgOperands == 1 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3557 I.getType()->isVectorTy() && I.onlyReadsMemory()) {
3558 // This looks like a vector load.
3559 return handleVectorLoadIntrinsic(I);
3560 }
3561
3562 if (I.doesNotAccessMemory())
3563 if (maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/0))
3564 return true;
3565
3566 // FIXME: detect and handle SSE maskstore/maskload?
3567 // Some cases are now handled in handleAVXMasked{Load,Store}.
3568 return false;
3569 }
3570
3571 bool maybeHandleUnknownIntrinsic(IntrinsicInst &I) {
3572 if (maybeHandleUnknownIntrinsicUnlogged(I)) {
3573 if (ClDumpHeuristicInstructions)
3574 dumpInst(I, Prefix: "Heuristic");
3575
3576 LLVM_DEBUG(dbgs() << "UNKNOWN INSTRUCTION HANDLED HEURISTICALLY: " << I
3577 << "\n");
3578 return true;
3579 } else
3580 return false;
3581 }
3582
3583 void handleInvariantGroup(IntrinsicInst &I) {
3584 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3585 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3586 }
3587
3588 void handleLifetimeStart(IntrinsicInst &I) {
3589 if (!PoisonStack)
3590 return;
3591 AllocaInst *AI = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
3592 if (AI)
3593 LifetimeStartList.push_back(Elt: std::make_pair(x: &I, y&: AI));
3594 }
3595
3596 void handleBswap(IntrinsicInst &I) {
3597 IRBuilder<> IRB(&I);
3598 Value *Op = I.getArgOperand(i: 0);
3599 Type *OpType = Op->getType();
3600 setShadow(V: &I, SV: IRB.CreateIntrinsic(ID: Intrinsic::bswap, OverloadTypes: ArrayRef(&OpType, 1),
3601 Args: getShadow(V: Op)));
3602 setOrigin(V: &I, Origin: getOrigin(V: Op));
3603 }
3604
3605 // Uninitialized bits are ok if they appear after the leading/trailing 0's
3606 // and a 1. If the input is all zero, it is fully initialized iff
3607 // !is_zero_poison.
3608 //
3609 // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
3610 // concrete value 0/1, and ? is an uninitialized bit:
3611 // - 0001 0??? is fully initialized
3612 // - 000? ???? is fully uninitialized (*)
3613 // - ???? ???? is fully uninitialized
3614 // - 0000 0000 is fully uninitialized if is_zero_poison,
3615 // fully initialized otherwise
3616 //
3617 // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
3618 // only need to poison 4 bits.
3619 //
3620 // OutputShadow =
3621 // ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
3622 // || (is_zero_poison && AllZeroSrc)
3623 void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
3624 IRBuilder<> IRB(&I);
3625 Value *Src = I.getArgOperand(i: 0);
3626 Value *SrcShadow = getShadow(V: Src);
3627
3628 Value *False = IRB.getInt1(V: false);
3629 Value *ConcreteZerosCount = IRB.CreateIntrinsic(
3630 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Src, /*is_zero_poison=*/False});
3631 Value *ShadowZerosCount = IRB.CreateIntrinsic(
3632 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {SrcShadow, /*is_zero_poison=*/False});
3633
3634 Value *CompareConcreteZeros = IRB.CreateICmpUGE(
3635 LHS: ConcreteZerosCount, RHS: ShadowZerosCount, Name: "_mscz_cmp_zeros");
3636
3637 Value *NotAllZeroShadow =
3638 IRB.CreateIsNotNull(Arg: SrcShadow, Name: "_mscz_shadow_not_null");
3639 Value *OutputShadow =
3640 IRB.CreateAnd(LHS: CompareConcreteZeros, RHS: NotAllZeroShadow, Name: "_mscz_main");
3641
3642 // If zero poison is requested, mix in with the shadow
3643 Constant *IsZeroPoison = cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3644 if (!IsZeroPoison->isNullValue()) {
3645 Value *BoolZeroPoison = IRB.CreateIsNull(Arg: Src, Name: "_mscz_bzp");
3646 OutputShadow = IRB.CreateOr(LHS: OutputShadow, RHS: BoolZeroPoison, Name: "_mscz_bs");
3647 }
3648
3649 OutputShadow = IRB.CreateSExt(V: OutputShadow, DestTy: getShadowTy(V: Src), Name: "_mscz_os");
3650
3651 setShadow(V: &I, SV: OutputShadow);
3652 setOriginForNaryOp(I);
3653 }
3654
3655 /// Some instructions have additional zero-elements in the return type
3656 /// e.g., <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, ...)
3657 ///
3658 /// This function will return a vector type with the same number of elements
3659 /// as the input, but same per-element width as the return value e.g.,
3660 /// <8 x i8>.
3661 FixedVectorType *maybeShrinkVectorShadowType(Value *Src, IntrinsicInst &I) {
3662 assert(isa<FixedVectorType>(getShadowTy(&I)));
3663 FixedVectorType *ShadowType = cast<FixedVectorType>(Val: getShadowTy(V: &I));
3664
3665 // TODO: generalize beyond 2x?
3666 if (ShadowType->getElementCount() ==
3667 cast<VectorType>(Val: Src->getType())->getElementCount() * 2)
3668 ShadowType = FixedVectorType::getHalfElementsVectorType(VTy: ShadowType);
3669
3670 assert(ShadowType->getElementCount() ==
3671 cast<VectorType>(Src->getType())->getElementCount());
3672
3673 return ShadowType;
3674 }
3675
3676 /// Doubles the length of a vector shadow (extending with zeros) if necessary
3677 /// to match the length of the shadow for the instruction.
3678 /// If scalar types of the vectors are different, it will use the type of the
3679 /// input vector.
3680 /// This is more type-safe than CreateShadowCast().
3681 Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
3682 IRBuilder<> IRB(&I);
3683 assert(isa<FixedVectorType>(Shadow->getType()));
3684 assert(isa<FixedVectorType>(I.getType()));
3685
3686 Value *FullShadow = getCleanShadow(V: &I);
3687 unsigned ShadowNumElems =
3688 cast<FixedVectorType>(Val: Shadow->getType())->getNumElements();
3689 unsigned FullShadowNumElems =
3690 cast<FixedVectorType>(Val: FullShadow->getType())->getNumElements();
3691
3692 assert((ShadowNumElems == FullShadowNumElems) ||
3693 (ShadowNumElems * 2 == FullShadowNumElems));
3694
3695 if (ShadowNumElems == FullShadowNumElems) {
3696 FullShadow = Shadow;
3697 } else {
3698 // TODO: generalize beyond 2x?
3699 SmallVector<int, 32> ShadowMask(FullShadowNumElems);
3700 std::iota(first: ShadowMask.begin(), last: ShadowMask.end(), value: 0);
3701
3702 // Append zeros
3703 FullShadow =
3704 IRB.CreateShuffleVector(V1: Shadow, V2: getCleanShadow(V: Shadow), Mask: ShadowMask);
3705 }
3706
3707 return FullShadow;
3708 }
3709
3710 /// Handle x86 SSE vector conversion.
3711 ///
3712 /// e.g., single-precision to half-precision conversion:
3713 /// <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
3714 /// <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0)
3715 ///
3716 /// floating-point to integer:
3717 /// <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>)
3718 /// <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>)
3719 ///
3720 /// Note: if the output has more elements, they are zero-initialized (and
3721 /// therefore the shadow will also be initialized).
3722 ///
3723 /// This differs from handleSSEVectorConvertIntrinsic() because it
3724 /// propagates uninitialized shadow (instead of checking the shadow).
3725 void handleSSEVectorConvertIntrinsicByProp(IntrinsicInst &I,
3726 bool HasRoundingMode) {
3727 if (HasRoundingMode) {
3728 assert(I.arg_size() == 2);
3729 [[maybe_unused]] Value *RoundingMode = I.getArgOperand(i: 1);
3730 assert(RoundingMode->getType()->isIntegerTy());
3731 } else {
3732 assert(I.arg_size() == 1);
3733 }
3734
3735 Value *Src = I.getArgOperand(i: 0);
3736 assert(Src->getType()->isVectorTy());
3737
3738 // The return type might have more elements than the input.
3739 // Temporarily shrink the return type's number of elements.
3740 VectorType *ShadowType = maybeShrinkVectorShadowType(Src, I);
3741
3742 IRBuilder<> IRB(&I);
3743 Value *S0 = getShadow(I: &I, i: 0);
3744
3745 /// For scalars:
3746 /// Since they are converting to and/or from floating-point, the output is:
3747 /// - fully uninitialized if *any* bit of the input is uninitialized
3748 /// - fully ininitialized if all bits of the input are ininitialized
3749 /// We apply the same principle on a per-field basis for vectors.
3750 Value *Shadow =
3751 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)), DestTy: ShadowType);
3752
3753 // The return type might have more elements than the input.
3754 // Extend the return type back to its original width if necessary.
3755 Value *FullShadow = maybeExtendVectorShadowWithZeros(Shadow, I);
3756
3757 setShadow(V: &I, SV: FullShadow);
3758 setOriginForNaryOp(I);
3759 }
3760
3761 // Instrument x86 SSE vector convert intrinsic.
3762 //
3763 // This function instruments intrinsics like cvtsi2ss:
3764 // %Out = int_xxx_cvtyyy(%ConvertOp)
3765 // or
3766 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
3767 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
3768 // number \p Out elements, and (if has 2 arguments) copies the rest of the
3769 // elements from \p CopyOp.
3770 // In most cases conversion involves floating-point value which may trigger a
3771 // hardware exception when not fully initialized. For this reason we require
3772 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
3773 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
3774 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
3775 // return a fully initialized value.
3776 //
3777 // For Arm NEON vector convert intrinsics, see
3778 // handleNEONVectorConvertIntrinsic().
3779 void handleSSEVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
3780 bool HasRoundingMode = false) {
3781 IRBuilder<> IRB(&I);
3782 Value *CopyOp, *ConvertOp;
3783
3784 assert((!HasRoundingMode ||
3785 isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
3786 "Invalid rounding mode");
3787
3788 switch (I.arg_size() - HasRoundingMode) {
3789 case 2:
3790 CopyOp = I.getArgOperand(i: 0);
3791 ConvertOp = I.getArgOperand(i: 1);
3792 break;
3793 case 1:
3794 ConvertOp = I.getArgOperand(i: 0);
3795 CopyOp = nullptr;
3796 break;
3797 default:
3798 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
3799 }
3800
3801 // The first *NumUsedElements* elements of ConvertOp are converted to the
3802 // same number of output elements. The rest of the output is copied from
3803 // CopyOp, or (if not available) filled with zeroes.
3804 // Combine shadow for elements of ConvertOp that are used in this operation,
3805 // and insert a check.
3806 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
3807 // int->any conversion.
3808 Value *ConvertShadow = getShadow(V: ConvertOp);
3809 Value *AggShadow = nullptr;
3810 if (ConvertOp->getType()->isVectorTy()) {
3811 AggShadow = IRB.CreateExtractElement(
3812 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
3813 for (int i = 1; i < NumUsedElements; ++i) {
3814 Value *MoreShadow = IRB.CreateExtractElement(
3815 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3816 AggShadow = IRB.CreateOr(LHS: AggShadow, RHS: MoreShadow);
3817 }
3818 } else {
3819 AggShadow = ConvertShadow;
3820 }
3821 assert(AggShadow->getType()->isIntegerTy());
3822 insertCheckShadow(Shadow: AggShadow, Origin: getOrigin(V: ConvertOp), OrigIns: &I);
3823
3824 // Build result shadow by zero-filling parts of CopyOp shadow that come from
3825 // ConvertOp.
3826 if (CopyOp) {
3827 assert(CopyOp->getType() == I.getType());
3828 assert(CopyOp->getType()->isVectorTy());
3829 Value *ResultShadow = getShadow(V: CopyOp);
3830 Type *EltTy = cast<VectorType>(Val: ResultShadow->getType())->getElementType();
3831 for (int i = 0; i < NumUsedElements; ++i) {
3832 ResultShadow = IRB.CreateInsertElement(
3833 Vec: ResultShadow, NewElt: ConstantInt::getNullValue(Ty: EltTy),
3834 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3835 }
3836 setShadow(V: &I, SV: ResultShadow);
3837 setOrigin(V: &I, Origin: getOrigin(V: CopyOp));
3838 } else {
3839 setShadow(V: &I, SV: getCleanShadow(V: &I));
3840 setOrigin(V: &I, Origin: getCleanOrigin());
3841 }
3842 }
3843
3844 // Given a scalar or vector, extract lower 64 bits (or less), and return all
3845 // zeroes if it is zero, and all ones otherwise.
3846 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3847 if (S->getType()->isVectorTy())
3848 S = CreateShadowCast(IRB, V: S, dstTy: IRB.getInt64Ty(), /* Signed */ true);
3849 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
3850 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3851 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3852 }
3853
3854 // Given a vector, extract its first element, and return all
3855 // zeroes if it is zero, and all ones otherwise.
3856 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3857 Value *S1 = IRB.CreateExtractElement(Vec: S, Idx: (uint64_t)0);
3858 Value *S2 = IRB.CreateICmpNE(LHS: S1, RHS: getCleanShadow(V: S1));
3859 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3860 }
3861
3862 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3863 Type *T = S->getType();
3864 assert(T->isVectorTy());
3865 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3866 return IRB.CreateSExt(V: S2, DestTy: T);
3867 }
3868
3869 // Instrument vector shift intrinsic.
3870 //
3871 // This function instruments intrinsics like int_x86_avx2_psll_w.
3872 // Intrinsic shifts %In by %ShiftSize bits.
3873 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
3874 // size, and the rest is ignored. Behavior is defined even if shift size is
3875 // greater than register (or field) width.
3876 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3877 assert(I.arg_size() == 2);
3878 IRBuilder<> IRB(&I);
3879 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3880 // Otherwise perform the same shift on S1.
3881 Value *S1 = getShadow(I: &I, i: 0);
3882 Value *S2 = getShadow(I: &I, i: 1);
3883 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S: S2)
3884 : Lower64ShadowExtend(IRB, S: S2, T: getShadowTy(V: &I));
3885 Value *V1 = I.getOperand(i_nocapture: 0);
3886 Value *V2 = I.getOperand(i_nocapture: 1);
3887 Value *Shift = IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
3888 Args: {IRB.CreateBitCast(V: S1, DestTy: V1->getType()), V2});
3889 Shift = IRB.CreateBitCast(V: Shift, DestTy: getShadowTy(V: &I));
3890 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3891 setOriginForNaryOp(I);
3892 }
3893
3894 // Get an MMX-sized (64-bit) vector type, or optionally, other sized
3895 // vectors.
3896 Type *getMMXVectorTy(unsigned EltSizeInBits,
3897 unsigned X86_MMXSizeInBits = 64) {
3898 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3899 "Illegal MMX vector element size");
3900 return FixedVectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSizeInBits),
3901 NumElts: X86_MMXSizeInBits / EltSizeInBits);
3902 }
3903
3904 // Returns a signed counterpart for an (un)signed-saturate-and-pack
3905 // intrinsic.
3906 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
3907 switch (id) {
3908 case Intrinsic::x86_sse2_packsswb_128:
3909 case Intrinsic::x86_sse2_packuswb_128:
3910 return Intrinsic::x86_sse2_packsswb_128;
3911
3912 case Intrinsic::x86_sse2_packssdw_128:
3913 case Intrinsic::x86_sse41_packusdw:
3914 return Intrinsic::x86_sse2_packssdw_128;
3915
3916 case Intrinsic::x86_avx2_packsswb:
3917 case Intrinsic::x86_avx2_packuswb:
3918 return Intrinsic::x86_avx2_packsswb;
3919
3920 case Intrinsic::x86_avx2_packssdw:
3921 case Intrinsic::x86_avx2_packusdw:
3922 return Intrinsic::x86_avx2_packssdw;
3923
3924 case Intrinsic::x86_mmx_packsswb:
3925 case Intrinsic::x86_mmx_packuswb:
3926 return Intrinsic::x86_mmx_packsswb;
3927
3928 case Intrinsic::x86_mmx_packssdw:
3929 return Intrinsic::x86_mmx_packssdw;
3930
3931 case Intrinsic::x86_avx512_packssdw_512:
3932 case Intrinsic::x86_avx512_packusdw_512:
3933 return Intrinsic::x86_avx512_packssdw_512;
3934
3935 case Intrinsic::x86_avx512_packsswb_512:
3936 case Intrinsic::x86_avx512_packuswb_512:
3937 return Intrinsic::x86_avx512_packsswb_512;
3938
3939 default:
3940 llvm_unreachable("unexpected intrinsic id");
3941 }
3942 }
3943
3944 // Instrument vector pack intrinsic.
3945 //
3946 // This function instruments intrinsics like x86_mmx_packsswb, that
3947 // packs elements of 2 input vectors into half as many bits with saturation.
3948 // Shadow is propagated with the signed variant of the same intrinsic applied
3949 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3950 // MMXEltSizeInBits is used only for x86mmx arguments.
3951 //
3952 // TODO: consider using GetMinMaxUnsigned() to handle saturation precisely
3953 void handleVectorPackIntrinsic(IntrinsicInst &I,
3954 unsigned MMXEltSizeInBits = 0) {
3955 assert(I.arg_size() == 2);
3956 IRBuilder<> IRB(&I);
3957 Value *S1 = getShadow(I: &I, i: 0);
3958 Value *S2 = getShadow(I: &I, i: 1);
3959 assert(S1->getType()->isVectorTy());
3960
3961 // SExt and ICmpNE below must apply to individual elements of input vectors.
3962 // In case of x86mmx arguments, cast them to appropriate vector types and
3963 // back.
3964 Type *T =
3965 MMXEltSizeInBits ? getMMXVectorTy(EltSizeInBits: MMXEltSizeInBits) : S1->getType();
3966 if (MMXEltSizeInBits) {
3967 S1 = IRB.CreateBitCast(V: S1, DestTy: T);
3968 S2 = IRB.CreateBitCast(V: S2, DestTy: T);
3969 }
3970 Value *S1_ext =
3971 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S1, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3972 Value *S2_ext =
3973 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3974 if (MMXEltSizeInBits) {
3975 S1_ext = IRB.CreateBitCast(V: S1_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3976 S2_ext = IRB.CreateBitCast(V: S2_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3977 }
3978
3979 Value *S = IRB.CreateIntrinsic(ID: getSignedPackIntrinsic(id: I.getIntrinsicID()),
3980 Args: {S1_ext, S2_ext}, /*FMFSource=*/nullptr,
3981 Name: "_msprop_vector_pack");
3982 if (MMXEltSizeInBits)
3983 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3984 setShadow(V: &I, SV: S);
3985 setOriginForNaryOp(I);
3986 }
3987
3988 // Convert `Mask` into `<n x i1>`.
3989 Constant *createDppMask(unsigned Width, unsigned Mask) {
3990 SmallVector<Constant *, 4> R(Width);
3991 for (auto &M : R) {
3992 M = ConstantInt::getBool(Context&: F.getContext(), V: Mask & 1);
3993 Mask >>= 1;
3994 }
3995 return ConstantVector::get(V: R);
3996 }
3997
3998 // Calculate output shadow as array of booleans `<n x i1>`, assuming if any
3999 // arg is poisoned, entire dot product is poisoned.
4000 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
4001 unsigned DstMask) {
4002 const unsigned Width =
4003 cast<FixedVectorType>(Val: S->getType())->getNumElements();
4004
4005 S = IRB.CreateSelect(C: createDppMask(Width, Mask: SrcMask), True: S,
4006 False: Constant::getNullValue(Ty: S->getType()));
4007 Value *SElem = IRB.CreateOrReduce(Src: S);
4008 Value *IsClean = IRB.CreateIsNull(Arg: SElem, Name: "_msdpp");
4009 Value *DstMaskV = createDppMask(Width, Mask: DstMask);
4010
4011 return IRB.CreateSelect(
4012 C: IsClean, True: Constant::getNullValue(Ty: DstMaskV->getType()), False: DstMaskV);
4013 }
4014
4015 // See `Intel Intrinsics Guide` for `_dp_p*` instructions.
4016 //
4017 // 2 and 4 element versions produce single scalar of dot product, and then
4018 // puts it into elements of output vector, selected by 4 lowest bits of the
4019 // mask. Top 4 bits of the mask control which elements of input to use for dot
4020 // product.
4021 //
4022 // 8 element version mask still has only 4 bit for input, and 4 bit for output
4023 // mask. According to the spec it just operates as 4 element version on first
4024 // 4 elements of inputs and output, and then on last 4 elements of inputs and
4025 // output.
4026 void handleDppIntrinsic(IntrinsicInst &I) {
4027 IRBuilder<> IRB(&I);
4028
4029 Value *S0 = getShadow(I: &I, i: 0);
4030 Value *S1 = getShadow(I: &I, i: 1);
4031 Value *S = IRB.CreateOr(LHS: S0, RHS: S1);
4032
4033 const unsigned Width =
4034 cast<FixedVectorType>(Val: S->getType())->getNumElements();
4035 assert(Width == 2 || Width == 4 || Width == 8);
4036
4037 const unsigned Mask = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4038 const unsigned SrcMask = Mask >> 4;
4039 const unsigned DstMask = Mask & 0xf;
4040
4041 // Calculate shadow as `<n x i1>`.
4042 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
4043 if (Width == 8) {
4044 // First 4 elements of shadow are already calculated. `makeDppShadow`
4045 // operats on 32 bit masks, so we can just shift masks, and repeat.
4046 SI1 = IRB.CreateOr(
4047 LHS: SI1, RHS: findDppPoisonedOutput(IRB, S, SrcMask: SrcMask << 4, DstMask: DstMask << 4));
4048 }
4049 // Extend to real size of shadow, poisoning either all or none bits of an
4050 // element.
4051 S = IRB.CreateSExt(V: SI1, DestTy: S->getType(), Name: "_msdpp");
4052
4053 setShadow(V: &I, SV: S);
4054 setOriginForNaryOp(I);
4055 }
4056
4057 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
4058 C = CreateAppToShadowCast(IRB, V: C);
4059 FixedVectorType *FVT = cast<FixedVectorType>(Val: C->getType());
4060 unsigned ElSize = FVT->getElementType()->getPrimitiveSizeInBits();
4061 C = IRB.CreateAShr(LHS: C, RHS: ElSize - 1);
4062 FVT = FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: FVT->getNumElements());
4063 return IRB.CreateTrunc(V: C, DestTy: FVT);
4064 }
4065
4066 // `blendv(f, t, c)` is effectively `select(c[top_bit], t, f)`.
4067 void handleBlendvIntrinsic(IntrinsicInst &I) {
4068 Value *C = I.getOperand(i_nocapture: 2);
4069 Value *T = I.getOperand(i_nocapture: 1);
4070 Value *F = I.getOperand(i_nocapture: 0);
4071
4072 Value *Sc = getShadow(I: &I, i: 2);
4073 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
4074
4075 {
4076 IRBuilder<> IRB(&I);
4077 // Extract top bit from condition and its shadow.
4078 C = convertBlendvToSelectMask(IRB, C);
4079 Sc = convertBlendvToSelectMask(IRB, C: Sc);
4080
4081 setShadow(V: C, SV: Sc);
4082 setOrigin(V: C, Origin: Oc);
4083 }
4084
4085 handleSelectLikeInst(I, B: C, C: T, D: F);
4086 }
4087
4088 // Instrument sum-of-absolute-differences intrinsic.
4089 void handleVectorSadIntrinsic(IntrinsicInst &I, bool IsMMX = false) {
4090 const unsigned SignificantBitsPerResultElement = 16;
4091 Type *ResTy = IsMMX ? IntegerType::get(C&: *MS.C, NumBits: 64) : I.getType();
4092 unsigned ZeroBitsPerResultElement =
4093 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
4094
4095 IRBuilder<> IRB(&I);
4096 auto *Shadow0 = getShadow(I: &I, i: 0);
4097 auto *Shadow1 = getShadow(I: &I, i: 1);
4098 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4099 S = IRB.CreateBitCast(V: S, DestTy: ResTy);
4100 S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
4101 DestTy: ResTy);
4102 S = IRB.CreateLShr(LHS: S, RHS: ZeroBitsPerResultElement);
4103 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
4104 setShadow(V: &I, SV: S);
4105 setOriginForNaryOp(I);
4106 }
4107
4108 // Instrument dot-product / multiply-add(-accumulate)? intrinsics.
4109 //
4110 // e.g., Two operands:
4111 // <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> %b)
4112 //
4113 // Two operands which require an EltSizeInBits override:
4114 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %a, <1 x i64> %b)
4115 //
4116 // Three operands:
4117 // <4 x i32> @llvm.x86.avx512.vpdpbusd.128
4118 // (<4 x i32> %s, <16 x i8> %a, <16 x i8> %b)
4119 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
4120 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
4121 // (these are equivalent to multiply-add on %a and %b, followed by
4122 // adding/"accumulating" %s. "Accumulation" stores the result in one
4123 // of the source registers, but this accumulate vs. add distinction
4124 // is lost when dealing with LLVM intrinsics.)
4125 //
4126 // ZeroPurifies means that multiplying a known-zero with an uninitialized
4127 // value results in an initialized value. This is applicable for integer
4128 // multiplication, but not floating-point (counter-example: NaN).
4129 void handleVectorDotProductIntrinsic(IntrinsicInst &I,
4130 unsigned ReductionFactor,
4131 bool ZeroPurifies,
4132 unsigned EltSizeInBits,
4133 enum OddOrEvenLanes Lanes) {
4134 IRBuilder<> IRB(&I);
4135
4136 [[maybe_unused]] FixedVectorType *ReturnType =
4137 cast<FixedVectorType>(Val: I.getType());
4138 assert(isa<FixedVectorType>(ReturnType));
4139
4140 // Vectors A and B, and shadows
4141 Value *Va = nullptr;
4142 Value *Vb = nullptr;
4143 Value *Sa = nullptr;
4144 Value *Sb = nullptr;
4145
4146 assert(I.arg_size() == 2 || I.arg_size() == 3);
4147 if (I.arg_size() == 2) {
4148 assert(Lanes == kBothLanes);
4149
4150 Va = I.getOperand(i_nocapture: 0);
4151 Vb = I.getOperand(i_nocapture: 1);
4152
4153 Sa = getShadow(I: &I, i: 0);
4154 Sb = getShadow(I: &I, i: 1);
4155 } else if (I.arg_size() == 3) {
4156 // Operand 0 is the accumulator. We will deal with that below.
4157 Va = I.getOperand(i_nocapture: 1);
4158 Vb = I.getOperand(i_nocapture: 2);
4159
4160 Sa = getShadow(I: &I, i: 1);
4161 Sb = getShadow(I: &I, i: 2);
4162
4163 if (Lanes == kEvenLanes || Lanes == kOddLanes) {
4164 // Convert < S0, S1, S2, S3, S4, S5, S6, S7 >
4165 // to < S0, S0, S2, S2, S4, S4, S6, S6 > (if even)
4166 // to < S1, S1, S3, S3, S5, S5, S7, S7 > (if odd)
4167 //
4168 // Note: for aarch64.neon.bfmlalb/t, the odd/even-indexed values are
4169 // zeroed, not duplicated. However, for shadow propagation, this
4170 // distinction is unimportant because Step 1 below will squeeze
4171 // each pair of elements (e.g., [S0, S0]) into a single bit, and
4172 // we only care if it is fully initialized.
4173
4174 FixedVectorType *InputShadowType = cast<FixedVectorType>(Val: Sa->getType());
4175 unsigned Width = InputShadowType->getNumElements();
4176
4177 Sa = IRB.CreateShuffleVector(
4178 V: Sa, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4179 Sb = IRB.CreateShuffleVector(
4180 V: Sb, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4181 }
4182 }
4183
4184 FixedVectorType *ParamType = cast<FixedVectorType>(Val: Va->getType());
4185 assert(ParamType == Vb->getType());
4186
4187 assert(ParamType->getPrimitiveSizeInBits() ==
4188 ReturnType->getPrimitiveSizeInBits());
4189
4190 if (I.arg_size() == 3) {
4191 [[maybe_unused]] auto *AccumulatorType =
4192 cast<FixedVectorType>(Val: I.getOperand(i_nocapture: 0)->getType());
4193 assert(AccumulatorType == ReturnType);
4194 }
4195
4196 FixedVectorType *ImplicitReturnType =
4197 cast<FixedVectorType>(Val: getShadowTy(OrigTy: ReturnType));
4198 // Step 1: instrument multiplication of corresponding vector elements
4199 if (EltSizeInBits) {
4200 ImplicitReturnType = cast<FixedVectorType>(
4201 Val: getMMXVectorTy(EltSizeInBits: EltSizeInBits * ReductionFactor,
4202 X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4203 ParamType = cast<FixedVectorType>(
4204 Val: getMMXVectorTy(EltSizeInBits, X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4205
4206 Va = IRB.CreateBitCast(V: Va, DestTy: ParamType);
4207 Vb = IRB.CreateBitCast(V: Vb, DestTy: ParamType);
4208
4209 Sa = IRB.CreateBitCast(V: Sa, DestTy: getShadowTy(OrigTy: ParamType));
4210 Sb = IRB.CreateBitCast(V: Sb, DestTy: getShadowTy(OrigTy: ParamType));
4211 } else {
4212 assert(ParamType->getNumElements() ==
4213 ReturnType->getNumElements() * ReductionFactor);
4214 }
4215
4216 // Each element of the vector is represented by a single bit (poisoned or
4217 // not) e.g., <8 x i1>.
4218 Value *SaNonZero = IRB.CreateIsNotNull(Arg: Sa);
4219 Value *SbNonZero = IRB.CreateIsNotNull(Arg: Sb);
4220 Value *And;
4221 if (ZeroPurifies) {
4222 // Multiplying an *initialized* zero by an uninitialized element results
4223 // in an initialized zero element.
4224 //
4225 // This is analogous to bitwise AND, where "AND" of 0 and a poisoned value
4226 // results in an unpoisoned value.
4227 Value *VaInt = Va;
4228 Value *VbInt = Vb;
4229 if (!Va->getType()->isIntegerTy()) {
4230 VaInt = CreateAppToShadowCast(IRB, V: Va);
4231 VbInt = CreateAppToShadowCast(IRB, V: Vb);
4232 }
4233
4234 // We check for non-zero on a per-element basis, not per-bit.
4235 Value *VaNonZero = IRB.CreateIsNotNull(Arg: VaInt);
4236 Value *VbNonZero = IRB.CreateIsNotNull(Arg: VbInt);
4237
4238 And = handleBitwiseAnd(IRB, V1: VaNonZero, V2: VbNonZero, S1: SaNonZero, S2: SbNonZero);
4239 } else {
4240 And = IRB.CreateOr(Ops: {SaNonZero, SbNonZero});
4241 }
4242
4243 // Extend <8 x i1> to <8 x i16>.
4244 // (The real pmadd intrinsic would have computed intermediate values of
4245 // <8 x i32>, but that is irrelevant for our shadow purposes because we
4246 // consider each element to be either fully initialized or fully
4247 // uninitialized.)
4248 And = IRB.CreateSExt(V: And, DestTy: Sa->getType());
4249
4250 // Step 2: instrument horizontal add
4251 // We don't need bit-precise horizontalReduce because we only want to check
4252 // if each pair/quad of elements is fully zero.
4253 // Cast to <4 x i32>.
4254 Value *Horizontal = IRB.CreateBitCast(V: And, DestTy: ImplicitReturnType);
4255
4256 // Compute <4 x i1>, then extend back to <4 x i32>.
4257 Value *OutShadow = IRB.CreateSExt(
4258 V: IRB.CreateICmpNE(LHS: Horizontal,
4259 RHS: Constant::getNullValue(Ty: Horizontal->getType())),
4260 DestTy: ImplicitReturnType);
4261
4262 // Cast it back to the required fake return type (if MMX: <1 x i64>; for
4263 // AVX, it is already correct).
4264 if (EltSizeInBits)
4265 OutShadow = CreateShadowCast(IRB, V: OutShadow, dstTy: getShadowTy(V: &I));
4266
4267 // Step 3 (if applicable): instrument accumulator
4268 if (I.arg_size() == 3)
4269 OutShadow = IRB.CreateOr(LHS: OutShadow, RHS: getShadow(I: &I, i: 0));
4270
4271 setShadow(V: &I, SV: OutShadow);
4272 setOriginForNaryOp(I);
4273 }
4274
4275 // Instrument compare-packed intrinsic.
4276 //
4277 // x86 has the predicate as the third operand, which is ImmArg e.g.,
4278 // - <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8)
4279 // - <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8)
4280 //
4281 // while Arm has separate intrinsics for >= and > e.g.,
4282 // - <2 x i32> @llvm.aarch64.neon.facge.v2i32.v2f32
4283 // (<2 x float> %A, <2 x float>)
4284 // - <2 x i32> @llvm.aarch64.neon.facgt.v2i32.v2f32
4285 // (<2 x float> %A, <2 x float>)
4286 //
4287 // Bonus: this also handles scalar cases e.g.,
4288 // - i32 @llvm.aarch64.neon.facgt.i32.f32(float %A, float %B)
4289 void handleVectorComparePackedIntrinsic(IntrinsicInst &I,
4290 bool PredicateAsOperand) {
4291 if (PredicateAsOperand) {
4292 assert(I.arg_size() == 3);
4293 assert(I.paramHasAttr(2, Attribute::ImmArg));
4294 } else
4295 assert(I.arg_size() == 2);
4296
4297 IRBuilder<> IRB(&I);
4298
4299 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
4300 // all-ones shadow.
4301 Type *ResTy = getShadowTy(V: &I);
4302 auto *Shadow0 = getShadow(I: &I, i: 0);
4303 auto *Shadow1 = getShadow(I: &I, i: 1);
4304 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4305 Value *S = IRB.CreateSExt(
4306 V: IRB.CreateICmpNE(LHS: S0, RHS: Constant::getNullValue(Ty: ResTy)), DestTy: ResTy);
4307 setShadow(V: &I, SV: S);
4308 setOriginForNaryOp(I);
4309 }
4310
4311 // Instrument compare-scalar intrinsic.
4312 // This handles both cmp* intrinsics which return the result in the first
4313 // element of a vector, and comi* which return the result as i32.
4314 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
4315 IRBuilder<> IRB(&I);
4316 auto *Shadow0 = getShadow(I: &I, i: 0);
4317 auto *Shadow1 = getShadow(I: &I, i: 1);
4318 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4319 Value *S = LowerElementShadowExtend(IRB, S: S0, T: getShadowTy(V: &I));
4320 setShadow(V: &I, SV: S);
4321 setOriginForNaryOp(I);
4322 }
4323
4324 // Instrument generic vector reduction intrinsics
4325 // by ORing together all their fields.
4326 //
4327 // If AllowShadowCast is true, the return type does not need to be the same
4328 // type as the fields
4329 // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
4330 void handleVectorReduceIntrinsic(IntrinsicInst &I, bool AllowShadowCast) {
4331 assert(I.arg_size() == 1);
4332
4333 IRBuilder<> IRB(&I);
4334 Value *S = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 0));
4335 if (AllowShadowCast)
4336 S = CreateShadowCast(IRB, V: S, dstTy: getShadowTy(V: &I));
4337 else
4338 assert(S->getType() == getShadowTy(&I));
4339 setShadow(V: &I, SV: S);
4340 setOriginForNaryOp(I);
4341 }
4342
4343 // Similar to handleVectorReduceIntrinsic but with an initial starting value.
4344 // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
4345 // %a1)
4346 // shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
4347 //
4348 // The type of the return value, initial starting value, and elements of the
4349 // vector must be identical.
4350 void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
4351 assert(I.arg_size() == 2);
4352
4353 IRBuilder<> IRB(&I);
4354 Value *Shadow0 = getShadow(I: &I, i: 0);
4355 Value *Shadow1 = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 1));
4356 assert(Shadow0->getType() == Shadow1->getType());
4357 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4358 assert(S->getType() == getShadowTy(&I));
4359 setShadow(V: &I, SV: S);
4360 setOriginForNaryOp(I);
4361 }
4362
4363 // Instrument vector.reduce.or intrinsic.
4364 // Valid (non-poisoned) set bits in the operand pull low the
4365 // corresponding shadow bits.
4366 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
4367 assert(I.arg_size() == 1);
4368
4369 IRBuilder<> IRB(&I);
4370 Value *OperandShadow = getShadow(I: &I, i: 0);
4371 Value *OperandUnsetBits = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
4372 Value *OperandUnsetOrPoison = IRB.CreateOr(LHS: OperandUnsetBits, RHS: OperandShadow);
4373 // Bit N is clean if any field's bit N is 1 and unpoison
4374 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandUnsetOrPoison);
4375 // Otherwise, it is clean if every field's bit N is unpoison
4376 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4377 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4378
4379 setShadow(V: &I, SV: S);
4380 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4381 }
4382
4383 // Instrument vector.reduce.and intrinsic.
4384 // Valid (non-poisoned) unset bits in the operand pull down the
4385 // corresponding shadow bits.
4386 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
4387 assert(I.arg_size() == 1);
4388
4389 IRBuilder<> IRB(&I);
4390 Value *OperandShadow = getShadow(I: &I, i: 0);
4391 Value *OperandSetOrPoison = IRB.CreateOr(LHS: I.getOperand(i_nocapture: 0), RHS: OperandShadow);
4392 // Bit N is clean if any field's bit N is 0 and unpoison
4393 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandSetOrPoison);
4394 // Otherwise, it is clean if every field's bit N is unpoison
4395 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4396 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4397
4398 setShadow(V: &I, SV: S);
4399 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4400 }
4401
4402 void handleStmxcsr(IntrinsicInst &I) {
4403 IRBuilder<> IRB(&I);
4404 Value *Addr = I.getArgOperand(i: 0);
4405 Type *Ty = IRB.getInt32Ty();
4406 Value *ShadowPtr =
4407 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment: Align(1), /*isStore*/ true).first;
4408
4409 IRB.CreateStore(Val: getCleanShadow(OrigTy: Ty), Ptr: ShadowPtr);
4410
4411 if (ClCheckAccessAddress)
4412 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4413 }
4414
4415 void handleLdmxcsr(IntrinsicInst &I) {
4416 if (!InsertChecks)
4417 return;
4418
4419 IRBuilder<> IRB(&I);
4420 Value *Addr = I.getArgOperand(i: 0);
4421 Type *Ty = IRB.getInt32Ty();
4422 const Align Alignment = Align(1);
4423 Value *ShadowPtr, *OriginPtr;
4424 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4425 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment, /*isStore*/ false);
4426
4427 if (ClCheckAccessAddress)
4428 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4429
4430 Value *Shadow = IRB.CreateAlignedLoad(Ty, Ptr: ShadowPtr, Align: Alignment, Name: "_ldmxcsr");
4431 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr)
4432 : getCleanOrigin();
4433 insertCheckShadow(Shadow, Origin, OrigIns: &I);
4434 }
4435
4436 void handleMaskedExpandLoad(IntrinsicInst &I) {
4437 IRBuilder<> IRB(&I);
4438 Value *Ptr = I.getArgOperand(i: 0);
4439 MaybeAlign Align = I.getParamAlign(ArgNo: 0);
4440 Value *Mask = I.getArgOperand(i: 1);
4441 Value *PassThru = I.getArgOperand(i: 2);
4442
4443 if (ClCheckAccessAddress) {
4444 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4445 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4446 }
4447
4448 if (!PropagateShadow) {
4449 setShadow(V: &I, SV: getCleanShadow(V: &I));
4450 setOrigin(V: &I, Origin: getCleanOrigin());
4451 return;
4452 }
4453
4454 Type *ShadowTy = getShadowTy(V: &I);
4455 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4456 auto [ShadowPtr, OriginPtr] =
4457 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ false);
4458
4459 Value *Shadow =
4460 IRB.CreateMaskedExpandLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align, Mask,
4461 PassThru: getShadow(V: PassThru), Name: "_msmaskedexpload");
4462
4463 setShadow(V: &I, SV: Shadow);
4464
4465 // TODO: Store origins.
4466 setOrigin(V: &I, Origin: getCleanOrigin());
4467 }
4468
4469 void handleMaskedCompressStore(IntrinsicInst &I) {
4470 IRBuilder<> IRB(&I);
4471 Value *Values = I.getArgOperand(i: 0);
4472 Value *Ptr = I.getArgOperand(i: 1);
4473 MaybeAlign Align = I.getParamAlign(ArgNo: 1);
4474 Value *Mask = I.getArgOperand(i: 2);
4475
4476 if (ClCheckAccessAddress) {
4477 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4478 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4479 }
4480
4481 Value *Shadow = getShadow(V: Values);
4482 Type *ElementShadowTy =
4483 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4484 auto [ShadowPtr, OriginPtrs] =
4485 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ true);
4486
4487 IRB.CreateMaskedCompressStore(Val: Shadow, Ptr: ShadowPtr, Align, Mask);
4488
4489 // TODO: Store origins.
4490 }
4491
4492 void handleMaskedGather(IntrinsicInst &I) {
4493 IRBuilder<> IRB(&I);
4494 Value *Ptrs = I.getArgOperand(i: 0);
4495 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4496 Value *Mask = I.getArgOperand(i: 1);
4497 Value *PassThru = I.getArgOperand(i: 2);
4498
4499 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4500 if (ClCheckAccessAddress) {
4501 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4502 Value *MaskedPtrShadow = IRB.CreateSelect(
4503 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4504 Name: "_msmaskedptrs");
4505 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4506 }
4507
4508 if (!PropagateShadow) {
4509 setShadow(V: &I, SV: getCleanShadow(V: &I));
4510 setOrigin(V: &I, Origin: getCleanOrigin());
4511 return;
4512 }
4513
4514 Type *ShadowTy = getShadowTy(V: &I);
4515 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4516 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4517 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ false);
4518
4519 Value *Shadow =
4520 IRB.CreateMaskedGather(Ty: ShadowTy, Ptrs: ShadowPtrs, Alignment, Mask,
4521 PassThru: getShadow(V: PassThru), Name: "_msmaskedgather");
4522
4523 setShadow(V: &I, SV: Shadow);
4524
4525 // TODO: Store origins.
4526 setOrigin(V: &I, Origin: getCleanOrigin());
4527 }
4528
4529 void handleMaskedScatter(IntrinsicInst &I) {
4530 IRBuilder<> IRB(&I);
4531 Value *Values = I.getArgOperand(i: 0);
4532 Value *Ptrs = I.getArgOperand(i: 1);
4533 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4534 Value *Mask = I.getArgOperand(i: 2);
4535
4536 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4537 if (ClCheckAccessAddress) {
4538 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4539 Value *MaskedPtrShadow = IRB.CreateSelect(
4540 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4541 Name: "_msmaskedptrs");
4542 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4543 }
4544
4545 Value *Shadow = getShadow(V: Values);
4546 Type *ElementShadowTy =
4547 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4548 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4549 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ true);
4550
4551 IRB.CreateMaskedScatter(Val: Shadow, Ptrs: ShadowPtrs, Alignment, Mask);
4552
4553 // TODO: Store origin.
4554 }
4555
4556 // Intrinsic::masked_store
4557 //
4558 // Note: handleAVXMaskedStore handles AVX/AVX2 variants, though AVX512 masked
4559 // stores are lowered to Intrinsic::masked_store.
4560 void handleMaskedStore(IntrinsicInst &I) {
4561 IRBuilder<> IRB(&I);
4562 Value *V = I.getArgOperand(i: 0);
4563 Value *Ptr = I.getArgOperand(i: 1);
4564 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4565 Value *Mask = I.getArgOperand(i: 2);
4566 Value *Shadow = getShadow(V);
4567
4568 if (ClCheckAccessAddress) {
4569 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4570 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4571 }
4572
4573 Value *ShadowPtr;
4574 Value *OriginPtr;
4575 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
4576 Addr: Ptr, IRB, ShadowTy: Shadow->getType(), Alignment, /*isStore*/ true);
4577
4578 IRB.CreateMaskedStore(Val: Shadow, Ptr: ShadowPtr, Alignment, Mask);
4579
4580 if (!MS.TrackOrigins)
4581 return;
4582
4583 auto &DL = F.getDataLayout();
4584 paintOrigin(IRB, Origin: getOrigin(V), OriginPtr,
4585 TS: DL.getTypeStoreSize(Ty: Shadow->getType()),
4586 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4587 }
4588
4589 // Intrinsic::masked_load
4590 //
4591 // Note: handleAVXMaskedLoad handles AVX/AVX2 variants, though AVX512 masked
4592 // loads are lowered to Intrinsic::masked_load.
4593 void handleMaskedLoad(IntrinsicInst &I) {
4594 IRBuilder<> IRB(&I);
4595 Value *Ptr = I.getArgOperand(i: 0);
4596 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4597 Value *Mask = I.getArgOperand(i: 1);
4598 Value *PassThru = I.getArgOperand(i: 2);
4599
4600 if (ClCheckAccessAddress) {
4601 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4602 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4603 }
4604
4605 if (!PropagateShadow) {
4606 setShadow(V: &I, SV: getCleanShadow(V: &I));
4607 setOrigin(V: &I, Origin: getCleanOrigin());
4608 return;
4609 }
4610
4611 Type *ShadowTy = getShadowTy(V: &I);
4612 Value *ShadowPtr, *OriginPtr;
4613 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4614 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
4615 setShadow(V: &I, SV: IRB.CreateMaskedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Alignment, Mask,
4616 PassThru: getShadow(V: PassThru), Name: "_msmaskedld"));
4617
4618 if (!MS.TrackOrigins)
4619 return;
4620
4621 // Choose between PassThru's and the loaded value's origins.
4622 Value *MaskedPassThruShadow = IRB.CreateAnd(
4623 LHS: getShadow(V: PassThru), RHS: IRB.CreateSExt(V: IRB.CreateNeg(V: Mask), DestTy: ShadowTy));
4624
4625 Value *NotNull = convertToBool(V: MaskedPassThruShadow, IRB, name: "_mscmp");
4626
4627 Value *PtrOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr);
4628 Value *Origin = IRB.CreateSelect(C: NotNull, True: getOrigin(V: PassThru), False: PtrOrigin);
4629
4630 setOrigin(V: &I, Origin);
4631 }
4632
4633 // e.g., void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)
4634 // dst mask src
4635 //
4636 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4637 // by handleMaskedStore.
4638 //
4639 // This function handles AVX and AVX2 masked stores; these use the MSBs of a
4640 // vector of integers, unlike the LLVM masked intrinsics, which require a
4641 // vector of booleans. X86InstCombineIntrinsic.cpp::simplifyX86MaskedLoad
4642 // mentions that the x86 backend does not know how to efficiently convert
4643 // from a vector of booleans back into the AVX mask format; therefore, they
4644 // (and we) do not reduce AVX/AVX2 masked intrinsics into LLVM masked
4645 // intrinsics.
4646 void handleAVXMaskedStore(IntrinsicInst &I) {
4647 assert(I.arg_size() == 3);
4648
4649 IRBuilder<> IRB(&I);
4650
4651 Value *Dst = I.getArgOperand(i: 0);
4652 assert(Dst->getType()->isPointerTy() && "Destination is not a pointer!");
4653
4654 Value *Mask = I.getArgOperand(i: 1);
4655 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4656
4657 Value *Src = I.getArgOperand(i: 2);
4658 assert(isa<VectorType>(Src->getType()) && "Source is not a vector!");
4659
4660 const Align Alignment = Align(1);
4661
4662 Value *SrcShadow = getShadow(V: Src);
4663
4664 if (ClCheckAccessAddress) {
4665 insertCheckShadowOf(Val: Dst, OrigIns: &I);
4666 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4667 }
4668
4669 Value *DstShadowPtr;
4670 Value *DstOriginPtr;
4671 std::tie(args&: DstShadowPtr, args&: DstOriginPtr) = getShadowOriginPtr(
4672 Addr: Dst, IRB, ShadowTy: SrcShadow->getType(), Alignment, /*isStore*/ true);
4673
4674 SmallVector<Value *, 2> ShadowArgs;
4675 ShadowArgs.append(NumInputs: 1, Elt: DstShadowPtr);
4676 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4677 // The intrinsic may require floating-point but shadows can be arbitrary
4678 // bit patterns, of which some would be interpreted as "invalid"
4679 // floating-point values (NaN etc.); we assume the intrinsic will happily
4680 // copy them.
4681 ShadowArgs.append(NumInputs: 1, Elt: IRB.CreateBitCast(V: SrcShadow, DestTy: Src->getType()));
4682
4683 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4684 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4685 setShadow(V: &I, SV: CI);
4686
4687 if (!MS.TrackOrigins)
4688 return;
4689
4690 // Approximation only
4691 auto &DL = F.getDataLayout();
4692 paintOrigin(IRB, Origin: getOrigin(V: Src), OriginPtr: DstOriginPtr,
4693 TS: DL.getTypeStoreSize(Ty: SrcShadow->getType()),
4694 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4695 }
4696
4697 // e.g., <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)
4698 // return src mask
4699 //
4700 // Masked-off values are replaced with 0, which conveniently also represents
4701 // initialized memory.
4702 //
4703 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4704 // by handleMaskedStore.
4705 //
4706 // We do not combine this with handleMaskedLoad; see comment in
4707 // handleAVXMaskedStore for the rationale.
4708 //
4709 // This is subtly different than handleIntrinsicByApplyingToShadow(I, 1)
4710 // because we need to apply getShadowOriginPtr, not getShadow, to the first
4711 // parameter.
4712 void handleAVXMaskedLoad(IntrinsicInst &I) {
4713 assert(I.arg_size() == 2);
4714
4715 IRBuilder<> IRB(&I);
4716
4717 Value *Src = I.getArgOperand(i: 0);
4718 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
4719
4720 Value *Mask = I.getArgOperand(i: 1);
4721 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4722
4723 const Align Alignment = Align(1);
4724
4725 if (ClCheckAccessAddress) {
4726 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4727 }
4728
4729 Type *SrcShadowTy = getShadowTy(V: Src);
4730 Value *SrcShadowPtr, *SrcOriginPtr;
4731 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
4732 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment, /*isStore*/ false);
4733
4734 SmallVector<Value *, 2> ShadowArgs;
4735 ShadowArgs.append(NumInputs: 1, Elt: SrcShadowPtr);
4736 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4737
4738 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4739 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4740 // The AVX masked load intrinsics do not have integer variants. We use the
4741 // floating-point variants, which will happily copy the shadows even if
4742 // they are interpreted as "invalid" floating-point values (NaN etc.).
4743 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4744
4745 if (!MS.TrackOrigins)
4746 return;
4747
4748 // The "pass-through" value is always zero (initialized). To the extent
4749 // that that results in initialized aligned 4-byte chunks, the origin value
4750 // is ignored. It is therefore correct to simply copy the origin from src.
4751 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
4752 setOrigin(V: &I, Origin: PtrSrcOrigin);
4753 }
4754
4755 // Test whether the mask indices are initialized, only checking the bits that
4756 // are actually used.
4757 //
4758 // e.g., if Idx is <32 x i16>, only (log2(32) == 5) bits of each index are
4759 // used/checked.
4760 void maskedCheckAVXIndexShadow(IRBuilder<> &IRB, Value *Idx, Instruction *I) {
4761 assert(isFixedIntVector(Idx));
4762 auto IdxVectorSize =
4763 cast<FixedVectorType>(Val: Idx->getType())->getNumElements();
4764 assert(isPowerOf2_64(IdxVectorSize));
4765
4766 // Compiler isn't smart enough, let's help it
4767 if (isa<Constant>(Val: Idx))
4768 return;
4769
4770 auto *IdxShadow = getShadow(V: Idx);
4771 Value *Truncated = IRB.CreateTrunc(
4772 V: IdxShadow,
4773 DestTy: FixedVectorType::get(ElementType: Type::getIntNTy(C&: *MS.C, N: Log2_64(Value: IdxVectorSize)),
4774 NumElts: IdxVectorSize));
4775 insertCheckShadow(Shadow: Truncated, Origin: getOrigin(V: Idx), OrigIns: I);
4776 }
4777
4778 // Instrument AVX permutation intrinsic.
4779 // We apply the same permutation (argument index 1) to the shadow.
4780 void handleAVXVpermilvar(IntrinsicInst &I) {
4781 IRBuilder<> IRB(&I);
4782 Value *Shadow = getShadow(I: &I, i: 0);
4783 maskedCheckAVXIndexShadow(IRB, Idx: I.getArgOperand(i: 1), I: &I);
4784
4785 // Shadows are integer-ish types but some intrinsics require a
4786 // different (e.g., floating-point) type.
4787 Shadow = IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i: 0)->getType());
4788 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4789 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Shadow, I.getArgOperand(i: 1)});
4790
4791 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4792 setOriginForNaryOp(I);
4793 }
4794
4795 // Instrument AVX permutation intrinsic.
4796 // We apply the same permutation (argument index 1) to the shadows.
4797 void handleAVXVpermi2var(IntrinsicInst &I) {
4798 assert(I.arg_size() == 3);
4799 assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
4800 assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
4801 assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
4802 [[maybe_unused]] auto ArgVectorSize =
4803 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4804 assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
4805 ->getNumElements() == ArgVectorSize);
4806 assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
4807 ->getNumElements() == ArgVectorSize);
4808 assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
4809 assert(I.getType() == I.getArgOperand(0)->getType());
4810 assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
4811 IRBuilder<> IRB(&I);
4812 Value *AShadow = getShadow(I: &I, i: 0);
4813 Value *Idx = I.getArgOperand(i: 1);
4814 Value *BShadow = getShadow(I: &I, i: 2);
4815
4816 maskedCheckAVXIndexShadow(IRB, Idx, I: &I);
4817
4818 // Shadows are integer-ish types but some intrinsics require a
4819 // different (e.g., floating-point) type.
4820 AShadow = IRB.CreateBitCast(V: AShadow, DestTy: I.getArgOperand(i: 0)->getType());
4821 BShadow = IRB.CreateBitCast(V: BShadow, DestTy: I.getArgOperand(i: 2)->getType());
4822 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4823 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {AShadow, Idx, BShadow});
4824 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4825 setOriginForNaryOp(I);
4826 }
4827
4828 [[maybe_unused]] static bool isFixedIntVectorTy(const Type *T) {
4829 return isa<FixedVectorType>(Val: T) && T->isIntOrIntVectorTy();
4830 }
4831
4832 [[maybe_unused]] static bool isFixedFPVectorTy(const Type *T) {
4833 return isa<FixedVectorType>(Val: T) && T->isFPOrFPVectorTy();
4834 }
4835
4836 [[maybe_unused]] static bool isFixedIntVector(const Value *V) {
4837 return isFixedIntVectorTy(T: V->getType());
4838 }
4839
4840 [[maybe_unused]] static bool isFixedFPVector(const Value *V) {
4841 return isFixedFPVectorTy(T: V->getType());
4842 }
4843
4844 // e.g., <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
4845 // (<16 x float> a, <16 x i32> writethru, i16 mask,
4846 // i32 rounding)
4847 //
4848 // Inconveniently, some similar intrinsics have a different operand order:
4849 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
4850 // (<16 x float> a, i32 rounding, <16 x i16> writethru,
4851 // i16 mask)
4852 //
4853 // If the return type has more elements than A, the excess elements are
4854 // zeroed (and the corresponding shadow is initialized).
4855 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
4856 // (<4 x float> a, i32 rounding, <8 x i16> writethru,
4857 // i8 mask)
4858 //
4859 // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
4860 // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
4861 // where all_or_nothing(x) is fully uninitialized if x has any
4862 // uninitialized bits
4863 void handleAVX512VectorConvertFPToInt(IntrinsicInst &I, bool LastMask) {
4864 IRBuilder<> IRB(&I);
4865
4866 assert(I.arg_size() == 4);
4867 Value *A = I.getOperand(i_nocapture: 0);
4868 Value *WriteThrough;
4869 Value *Mask;
4870 Value *RoundingMode;
4871 if (LastMask) {
4872 WriteThrough = I.getOperand(i_nocapture: 2);
4873 Mask = I.getOperand(i_nocapture: 3);
4874 RoundingMode = I.getOperand(i_nocapture: 1);
4875 } else {
4876 WriteThrough = I.getOperand(i_nocapture: 1);
4877 Mask = I.getOperand(i_nocapture: 2);
4878 RoundingMode = I.getOperand(i_nocapture: 3);
4879 }
4880
4881 assert(isFixedFPVector(A));
4882 assert(isFixedIntVector(WriteThrough));
4883
4884 unsigned ANumElements =
4885 cast<FixedVectorType>(Val: A->getType())->getNumElements();
4886 [[maybe_unused]] unsigned WriteThruNumElements =
4887 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
4888 assert(ANumElements == WriteThruNumElements ||
4889 ANumElements * 2 == WriteThruNumElements);
4890
4891 assert(Mask->getType()->isIntegerTy());
4892 unsigned MaskNumElements = Mask->getType()->getScalarSizeInBits();
4893 assert(ANumElements == MaskNumElements ||
4894 ANumElements * 2 == MaskNumElements);
4895
4896 assert(WriteThruNumElements == MaskNumElements);
4897
4898 // Some bits of the mask may be unused, though it's unusual to have partly
4899 // uninitialized bits.
4900 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4901
4902 assert(RoundingMode->getType()->isIntegerTy());
4903 // Only some bits of the rounding mode are used, though it's very
4904 // unusual to have uninitialized bits there (more commonly, it's a
4905 // constant).
4906 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
4907
4908 assert(I.getType() == WriteThrough->getType());
4909
4910 Value *AShadow = getShadow(V: A);
4911 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
4912
4913 if (ANumElements * 2 == MaskNumElements) {
4914 // Ensure that the irrelevant bits of the mask are zero, hence selecting
4915 // from the zeroed shadow instead of the writethrough's shadow.
4916 Mask =
4917 IRB.CreateTrunc(V: Mask, DestTy: IRB.getIntNTy(N: ANumElements), Name: "_ms_mask_trunc");
4918 Mask =
4919 IRB.CreateZExt(V: Mask, DestTy: IRB.getIntNTy(N: MaskNumElements), Name: "_ms_mask_zext");
4920 }
4921
4922 // Convert i16 mask to <16 x i1>
4923 Mask = IRB.CreateBitCast(
4924 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: MaskNumElements),
4925 Name: "_ms_mask_bitcast");
4926
4927 /// For floating-point to integer conversion, the output is:
4928 /// - fully uninitialized if *any* bit of the input is uninitialized
4929 /// - fully ininitialized if all bits of the input are ininitialized
4930 /// We apply the same principle on a per-element basis for vectors.
4931 ///
4932 /// We use the scalar width of the return type instead of A's.
4933 AShadow = IRB.CreateSExt(
4934 V: IRB.CreateICmpNE(LHS: AShadow, RHS: getCleanShadow(OrigTy: AShadow->getType())),
4935 DestTy: getShadowTy(V: &I), Name: "_ms_a_shadow");
4936
4937 Value *WriteThroughShadow = getShadow(V: WriteThrough);
4938 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow,
4939 Name: "_ms_writethru_select");
4940
4941 setShadow(V: &I, SV: Shadow);
4942 setOriginForNaryOp(I);
4943 }
4944
4945 static SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
4946 SmallVector<int, 8> Mask;
4947 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
4948 Mask.append(NumInputs: 2, Elt: X);
4949 }
4950 return Mask;
4951 }
4952
4953 // Instrument pclmul intrinsics.
4954 // These intrinsics operate either on odd or on even elements of the input
4955 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
4956 // Replace the unused elements with copies of the used ones, ex:
4957 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
4958 // or
4959 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
4960 // and then apply the usual shadow combining logic.
4961 void handlePclmulIntrinsic(IntrinsicInst &I) {
4962 IRBuilder<> IRB(&I);
4963 unsigned Width =
4964 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4965 assert(isa<ConstantInt>(I.getArgOperand(2)) &&
4966 "pclmul 3rd operand must be a constant");
4967 unsigned Imm = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4968 Value *Shuf0 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 0),
4969 Mask: getPclmulMask(Width, OddElements: Imm & 0x01));
4970 Value *Shuf1 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 1),
4971 Mask: getPclmulMask(Width, OddElements: Imm & 0x10));
4972 ShadowAndOriginCombiner SOC(this, IRB);
4973 SOC.Add(OpShadow: Shuf0, OpOrigin: getOrigin(I: &I, i: 0));
4974 SOC.Add(OpShadow: Shuf1, OpOrigin: getOrigin(I: &I, i: 1));
4975 SOC.Done(I: &I);
4976 }
4977
4978 // Instrument _mm_*_sd|ss intrinsics
4979 void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
4980 IRBuilder<> IRB(&I);
4981 unsigned Width =
4982 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4983 Value *First = getShadow(I: &I, i: 0);
4984 Value *Second = getShadow(I: &I, i: 1);
4985 // First element of second operand, remaining elements of first operand
4986 SmallVector<int, 16> Mask;
4987 Mask.push_back(Elt: Width);
4988 for (unsigned i = 1; i < Width; i++)
4989 Mask.push_back(Elt: i);
4990 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: Second, Mask);
4991
4992 setShadow(V: &I, SV: Shadow);
4993 setOriginForNaryOp(I);
4994 }
4995
4996 void handleVtestIntrinsic(IntrinsicInst &I) {
4997 IRBuilder<> IRB(&I);
4998 Value *Shadow0 = getShadow(I: &I, i: 0);
4999 Value *Shadow1 = getShadow(I: &I, i: 1);
5000 Value *Or = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
5001 Value *NZ = IRB.CreateICmpNE(LHS: Or, RHS: Constant::getNullValue(Ty: Or->getType()));
5002 Value *Scalar = convertShadowToScalar(V: NZ, IRB);
5003 Value *Shadow = IRB.CreateZExt(V: Scalar, DestTy: getShadowTy(V: &I));
5004
5005 setShadow(V: &I, SV: Shadow);
5006 setOriginForNaryOp(I);
5007 }
5008
5009 void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
5010 IRBuilder<> IRB(&I);
5011 unsigned Width =
5012 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
5013 Value *First = getShadow(I: &I, i: 0);
5014 Value *Second = getShadow(I: &I, i: 1);
5015 Value *OrShadow = IRB.CreateOr(LHS: First, RHS: Second);
5016 // First element of both OR'd together, remaining elements of first operand
5017 SmallVector<int, 16> Mask;
5018 Mask.push_back(Elt: Width);
5019 for (unsigned i = 1; i < Width; i++)
5020 Mask.push_back(Elt: i);
5021 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: OrShadow, Mask);
5022
5023 setShadow(V: &I, SV: Shadow);
5024 setOriginForNaryOp(I);
5025 }
5026
5027 // _mm_round_ps / _mm_round_ps.
5028 // Similar to maybeHandleSimpleNomemIntrinsic except
5029 // the second argument is guaranteed to be a constant integer.
5030 void handleRoundPdPsIntrinsic(IntrinsicInst &I) {
5031 assert(I.getArgOperand(0)->getType() == I.getType());
5032 assert(I.arg_size() == 2);
5033 assert(isa<ConstantInt>(I.getArgOperand(1)));
5034
5035 IRBuilder<> IRB(&I);
5036 ShadowAndOriginCombiner SC(this, IRB);
5037 SC.Add(V: I.getArgOperand(i: 0));
5038 SC.Done(I: &I);
5039 }
5040
5041 // Instrument @llvm.abs intrinsic.
5042 //
5043 // e.g., i32 @llvm.abs.i32 (i32 <Src>, i1 <is_int_min_poison>)
5044 // <4 x i32> @llvm.abs.v4i32(<4 x i32> <Src>, i1 <is_int_min_poison>)
5045 void handleAbsIntrinsic(IntrinsicInst &I) {
5046 assert(I.arg_size() == 2);
5047 Value *Src = I.getArgOperand(i: 0);
5048 Value *IsIntMinPoison = I.getArgOperand(i: 1);
5049
5050 assert(I.getType()->isIntOrIntVectorTy());
5051
5052 assert(Src->getType() == I.getType());
5053
5054 assert(IsIntMinPoison->getType()->isIntegerTy());
5055 assert(IsIntMinPoison->getType()->getIntegerBitWidth() == 1);
5056
5057 IRBuilder<> IRB(&I);
5058 Value *SrcShadow = getShadow(V: Src);
5059
5060 APInt MinVal =
5061 APInt::getSignedMinValue(numBits: Src->getType()->getScalarSizeInBits());
5062 Value *MinValVec = ConstantInt::get(Ty: Src->getType(), V: MinVal);
5063 Value *SrcIsMin = IRB.CreateICmp(P: CmpInst::ICMP_EQ, LHS: Src, RHS: MinValVec);
5064
5065 Value *PoisonedShadow = getPoisonedShadow(V: Src);
5066 Value *PoisonedIfIntMinShadow =
5067 IRB.CreateSelect(C: SrcIsMin, True: PoisonedShadow, False: SrcShadow);
5068 Value *Shadow =
5069 IRB.CreateSelect(C: IsIntMinPoison, True: PoisonedIfIntMinShadow, False: SrcShadow);
5070
5071 setShadow(V: &I, SV: Shadow);
5072 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5073 }
5074
5075 void handleIsFpClass(IntrinsicInst &I) {
5076 IRBuilder<> IRB(&I);
5077 Value *Shadow = getShadow(I: &I, i: 0);
5078 setShadow(V: &I, SV: IRB.CreateICmpNE(LHS: Shadow, RHS: getCleanShadow(V: Shadow)));
5079 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5080 }
5081
5082 void handleArithmeticWithOverflow(IntrinsicInst &I) {
5083 IRBuilder<> IRB(&I);
5084 Value *Shadow0 = getShadow(I: &I, i: 0);
5085 Value *Shadow1 = getShadow(I: &I, i: 1);
5086 Value *ShadowElt0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
5087 Value *ShadowElt1 =
5088 IRB.CreateICmpNE(LHS: ShadowElt0, RHS: getCleanShadow(V: ShadowElt0));
5089
5090 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5091 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt0, Idxs: 0);
5092 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt1, Idxs: 1);
5093
5094 setShadow(V: &I, SV: Shadow);
5095 setOriginForNaryOp(I);
5096 }
5097
5098 void handleModfOrSincos(IntrinsicInst &I) {
5099 IRBuilder<> IRB(&I);
5100 Value *ArgShadow = getShadow(I: &I, i: 0);
5101 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5102 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 0);
5103 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 1);
5104 setShadow(V: &I, SV: Shadow);
5105 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5106 }
5107
5108 Value *extractLowerShadow(IRBuilder<> &IRB, Value *V) {
5109 assert(isa<FixedVectorType>(V->getType()));
5110 assert(cast<FixedVectorType>(V->getType())->getNumElements() > 0);
5111 Value *Shadow = getShadow(V);
5112 return IRB.CreateExtractElement(Vec: Shadow,
5113 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5114 }
5115
5116 // Handle llvm.x86.avx512.mask.pmov{,s,us}.*.{128,256,512}
5117 //
5118 // e.g., call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512
5119 // (<8 x i64>, <16 x i8>, i8)
5120 // A WriteThru Mask
5121 //
5122 // call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512
5123 // (<16 x i32>, <16 x i8>, i16)
5124 //
5125 // Dst[i] = Mask[i] ? truncate_or_saturate(A[i]) : WriteThru[i]
5126 // Dst_shadow[i] = Mask[i] ? truncate(A_shadow[i]) : WriteThru_shadow[i]
5127 //
5128 // If Dst has more elements than A, the excess elements are zeroed (and the
5129 // corresponding shadow is initialized).
5130 //
5131 // Note: for PMOV (truncation), handleIntrinsicByApplyingToShadow is precise
5132 // and is much faster than this handler.
5133 void handleAVX512VectorDownConvert(IntrinsicInst &I) {
5134 IRBuilder<> IRB(&I);
5135
5136 assert(I.arg_size() == 3);
5137 Value *A = I.getOperand(i_nocapture: 0);
5138 Value *WriteThrough = I.getOperand(i_nocapture: 1);
5139 Value *Mask = I.getOperand(i_nocapture: 2);
5140
5141 assert(isFixedIntVector(A));
5142 assert(isFixedIntVector(WriteThrough));
5143
5144 unsigned ANumElements =
5145 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5146 unsigned OutputNumElements =
5147 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
5148 assert(ANumElements == OutputNumElements ||
5149 ANumElements * 2 == OutputNumElements);
5150 // N.B. some PMOV{,S,US} instructions have a 4x or even 8x ratio in the
5151 // number of elements e.g.,
5152 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256
5153 // (<4 x i64>, <16 x i8>, i8)
5154 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128
5155 // (<2 x i64>, <16 x i8>, i8)
5156 // However, we currently handle those elsewhere.
5157
5158 assert(Mask->getType()->isIntegerTy());
5159 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5160
5161 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5162 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5163 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5164 assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
5165
5166 assert(I.getType() == WriteThrough->getType());
5167
5168 // Widen the mask, if necessary, to have one bit per element of the output
5169 // vector.
5170 // We want the extra bits to have '1's, so that the CreateSelect will
5171 // select the values from AShadow instead of WriteThroughShadow ("maskless"
5172 // versions of the intrinsics are sometimes implemented using an all-1's
5173 // mask and an undefined value for WriteThroughShadow). We accomplish this
5174 // by using bitwise NOT before and after the ZExt.
5175 if (ANumElements != OutputNumElements) {
5176 Mask = IRB.CreateNot(V: Mask);
5177 Mask = IRB.CreateZExt(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements),
5178 Name: "_ms_widen_mask");
5179 Mask = IRB.CreateNot(V: Mask);
5180 }
5181 Mask = IRB.CreateBitCast(
5182 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5183
5184 Value *AShadow = getShadow(V: A);
5185
5186 // The return type might have more elements than the input.
5187 // Temporarily shrink the return type's number of elements.
5188 VectorType *ShadowType = maybeShrinkVectorShadowType(Src: A, I);
5189
5190 // PMOV truncates; PMOVS/PMOVUS uses signed/unsigned saturation.
5191 // This handler treats them all as truncation, which leads to some rare
5192 // false positives in the cases where the truncated bytes could
5193 // unambiguously saturate the value e.g., if A = ??????10 ????????
5194 // (big-endian), the unsigned saturated byte conversion is 11111111 i.e.,
5195 // fully defined, but the truncated byte is ????????.
5196 //
5197 // TODO: use GetMinMaxUnsigned() to handle saturation precisely.
5198 AShadow = IRB.CreateTrunc(V: AShadow, DestTy: ShadowType, Name: "_ms_trunc_shadow");
5199 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
5200
5201 Value *WriteThroughShadow = getShadow(V: WriteThrough);
5202
5203 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow);
5204 setShadow(V: &I, SV: Shadow);
5205 setOriginForNaryOp(I);
5206 }
5207
5208 // Handle llvm.x86.avx512.* instructions that take vector(s) of floating-point
5209 // values and perform an operation whose shadow propagation should be handled
5210 // as all-or-nothing [*], with masking provided by a vector and a mask
5211 // supplied as an integer.
5212 //
5213 // [*] if all bits of a vector element are initialized, the output is fully
5214 // initialized; otherwise, the output is fully uninitialized
5215 //
5216 // e.g., <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
5217 // (<16 x float>, <16 x float>, i16)
5218 // A WriteThru Mask
5219 //
5220 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
5221 // (<2 x double>, <2 x double>, i8)
5222 // A WriteThru Mask
5223 //
5224 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
5225 // (<8 x double>, i32, <8 x double>, i8, i32)
5226 // A Imm WriteThru Mask Rounding
5227 //
5228 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
5229 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
5230 // WriteThru A B Mask Rnd
5231 //
5232 // All operands other than A, B, ..., and WriteThru (e.g., Mask, Imm,
5233 // Rounding) must be fully initialized.
5234 //
5235 // Dst[i] = Mask[i] ? some_op(A[i], B[i], ...)
5236 // : WriteThru[i]
5237 // Dst_shadow[i] = Mask[i] ? all_or_nothing(A_shadow[i] | B_shadow[i] | ...)
5238 // : WriteThru_shadow[i]
5239 void handleAVX512VectorGenericMaskedFP(IntrinsicInst &I,
5240 SmallVector<unsigned, 4> DataIndices,
5241 unsigned WriteThruIndex,
5242 unsigned MaskIndex) {
5243 IRBuilder<> IRB(&I);
5244
5245 unsigned NumArgs = I.arg_size();
5246
5247 assert(WriteThruIndex < NumArgs);
5248 assert(MaskIndex < NumArgs);
5249 assert(WriteThruIndex != MaskIndex);
5250 Value *WriteThru = I.getOperand(i_nocapture: WriteThruIndex);
5251
5252 unsigned OutputNumElements =
5253 cast<FixedVectorType>(Val: WriteThru->getType())->getNumElements();
5254
5255 assert(DataIndices.size() > 0);
5256
5257 bool isData[16] = {false};
5258 assert(NumArgs <= 16);
5259 for (unsigned i : DataIndices) {
5260 assert(i < NumArgs);
5261 assert(i != WriteThruIndex);
5262 assert(i != MaskIndex);
5263
5264 isData[i] = true;
5265
5266 Value *A = I.getOperand(i_nocapture: i);
5267 assert(isFixedFPVector(A));
5268 [[maybe_unused]] unsigned ANumElements =
5269 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5270 assert(ANumElements == OutputNumElements);
5271 }
5272
5273 Value *Mask = I.getOperand(i_nocapture: MaskIndex);
5274
5275 assert(isFixedFPVector(WriteThru));
5276
5277 for (unsigned i = 0; i < NumArgs; ++i) {
5278 if (!isData[i] && i != WriteThruIndex) {
5279 // Imm, Mask, Rounding etc. are "control" data, hence we require that
5280 // they be fully initialized.
5281 assert(I.getOperand(i)->getType()->isIntegerTy());
5282 insertCheckShadowOf(Val: I.getOperand(i_nocapture: i), OrigIns: &I);
5283 }
5284 }
5285
5286 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5287 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5288 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5289 assert(Mask->getType()->getScalarSizeInBits() == OutputNumElements);
5290
5291 assert(I.getType() == WriteThru->getType());
5292
5293 Mask = IRB.CreateBitCast(
5294 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5295
5296 Value *DataShadow = nullptr;
5297 for (unsigned i : DataIndices) {
5298 Value *A = I.getOperand(i_nocapture: i);
5299 if (DataShadow)
5300 DataShadow = IRB.CreateOr(LHS: DataShadow, RHS: getShadow(V: A));
5301 else
5302 DataShadow = getShadow(V: A);
5303 }
5304
5305 // All-or-nothing shadow
5306 DataShadow =
5307 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: DataShadow, RHS: getCleanShadow(V: DataShadow)),
5308 DestTy: DataShadow->getType());
5309
5310 Value *WriteThruShadow = getShadow(V: WriteThru);
5311
5312 Value *Shadow = IRB.CreateSelect(C: Mask, True: DataShadow, False: WriteThruShadow);
5313 setShadow(V: &I, SV: Shadow);
5314
5315 setOriginForNaryOp(I);
5316 }
5317
5318 // AVX512 Floating-Point Classification
5319 //
5320 // e.g.,
5321 // - < 8 x i1> @llvm.x86.avx512.fpclass.pd.512
5322 // (<8 x double> %input, i32 %classifiers)
5323 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512
5324 // (<16 x float> %input, i32 %classifiers)
5325 void handleAVX512FPClass(IntrinsicInst &I) {
5326 IRBuilder<> IRB(&I);
5327
5328 assert(I.arg_size() == 2);
5329
5330 Value *Input = I.getOperand(i_nocapture: 0);
5331 assert(isFixedFPVector(Input));
5332 [[maybe_unused]] FixedVectorType *InputType = cast<FixedVectorType>(Val: Input->getType());
5333
5334 Value *Classifiers = I.getOperand(i_nocapture: 1);
5335 assert(isa<ConstantInt>(Classifiers));
5336 // No shadow check needed for constants
5337
5338 assert(isFixedIntVectorTy(I.getType()));
5339 FixedVectorType *OutputType = cast<FixedVectorType>(Val: I.getType());
5340 assert(OutputType->getScalarSizeInBits() == 1);
5341
5342 assert(OutputType->getNumElements() == InputType->getNumElements());
5343
5344 Value *OutputShadow;
5345 if (cast<ConstantInt>(Val: Classifiers)->isZero())
5346 // Each bit specifies whether a particular classifier is enabled.
5347 // If Classifiers == 0, the output is trivially known to be zero, thus
5348 // the output is fully initialized.
5349 OutputShadow = getCleanShadow(OrigTy: OutputType);
5350 else
5351 // Approximate each bit of the output shadow based on whether the
5352 // corresponding input element is fully initialized. It is only
5353 // approximate because some classifications do not rely on all bits of
5354 // the input element.
5355 OutputShadow = IRB.CreateICmpNE(LHS: getShadow(V: Input), RHS: getCleanShadow(V: Input));
5356
5357 setShadow(V: &I, SV: OutputShadow);
5358
5359 setOriginForNaryOp(I);
5360 }
5361
5362 // For sh.* compiler intrinsics:
5363 // llvm.x86.avx512fp16.mask.{add/sub/mul/div/max/min}.sh.round
5364 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
5365 // A B WriteThru Mask RoundingMode
5366 //
5367 // DstShadow[0] = Mask[0] ? (AShadow[0] | BShadow[0]) : WriteThruShadow[0]
5368 // DstShadow[1..7] = AShadow[1..7]
5369 void visitGenericScalarHalfwordInst(IntrinsicInst &I) {
5370 IRBuilder<> IRB(&I);
5371
5372 assert(I.arg_size() == 5);
5373 Value *A = I.getOperand(i_nocapture: 0);
5374 Value *B = I.getOperand(i_nocapture: 1);
5375 Value *WriteThrough = I.getOperand(i_nocapture: 2);
5376 Value *Mask = I.getOperand(i_nocapture: 3);
5377 Value *RoundingMode = I.getOperand(i_nocapture: 4);
5378
5379 // Technically, we could probably just check whether the LSB is
5380 // initialized, but intuitively it feels like a partly uninitialized mask
5381 // is unintended, and we should warn the user immediately.
5382 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5383 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
5384
5385 assert(isa<FixedVectorType>(A->getType()));
5386 unsigned NumElements =
5387 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5388 assert(NumElements == 8);
5389 assert(A->getType() == B->getType());
5390 assert(B->getType() == WriteThrough->getType());
5391 assert(Mask->getType()->getPrimitiveSizeInBits() == NumElements);
5392 assert(RoundingMode->getType()->isIntegerTy());
5393
5394 Value *ALowerShadow = extractLowerShadow(IRB, V: A);
5395 Value *BLowerShadow = extractLowerShadow(IRB, V: B);
5396
5397 Value *ABLowerShadow = IRB.CreateOr(LHS: ALowerShadow, RHS: BLowerShadow);
5398
5399 Value *WriteThroughLowerShadow = extractLowerShadow(IRB, V: WriteThrough);
5400
5401 Mask = IRB.CreateBitCast(
5402 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: NumElements));
5403 Value *MaskLower =
5404 IRB.CreateExtractElement(Vec: Mask, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5405
5406 Value *AShadow = getShadow(V: A);
5407 Value *DstLowerShadow =
5408 IRB.CreateSelect(C: MaskLower, True: ABLowerShadow, False: WriteThroughLowerShadow);
5409 Value *DstShadow = IRB.CreateInsertElement(
5410 Vec: AShadow, NewElt: DstLowerShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0),
5411 Name: "_msprop");
5412
5413 setShadow(V: &I, SV: DstShadow);
5414 setOriginForNaryOp(I);
5415 }
5416
5417 // Approximately handle AVX Galois Field Affine Transformation
5418 //
5419 // e.g.,
5420 // <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
5421 // <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8>, <32 x i8>, i8)
5422 // <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
5423 // Out A x b
5424 // where A and x are packed matrices, b is a vector,
5425 // Out = A * x + b in GF(2)
5426 //
5427 // Multiplication in GF(2) is equivalent to bitwise AND. However, the matrix
5428 // computation also includes a parity calculation.
5429 //
5430 // For the bitwise AND of bits V1 and V2, the exact shadow is:
5431 // Out_Shadow = (V1_Shadow & V2_Shadow)
5432 // | (V1 & V2_Shadow)
5433 // | (V1_Shadow & V2 )
5434 //
5435 // We approximate the shadow of gf2p8affineqb using:
5436 // Out_Shadow = gf2p8affineqb(x_Shadow, A_shadow, 0)
5437 // | gf2p8affineqb(x, A_shadow, 0)
5438 // | gf2p8affineqb(x_Shadow, A, 0)
5439 // | set1_epi8(b_Shadow)
5440 //
5441 // This approximation has false negatives: if an intermediate dot-product
5442 // contains an even number of 1's, the parity is 0.
5443 // It has no false positives.
5444 void handleAVXGF2P8Affine(IntrinsicInst &I) {
5445 IRBuilder<> IRB(&I);
5446
5447 assert(I.arg_size() == 3);
5448 Value *A = I.getOperand(i_nocapture: 0);
5449 Value *X = I.getOperand(i_nocapture: 1);
5450 Value *B = I.getOperand(i_nocapture: 2);
5451
5452 assert(isFixedIntVector(A));
5453 assert(cast<VectorType>(A->getType())
5454 ->getElementType()
5455 ->getScalarSizeInBits() == 8);
5456
5457 assert(A->getType() == X->getType());
5458
5459 assert(B->getType()->isIntegerTy());
5460 assert(B->getType()->getScalarSizeInBits() == 8);
5461
5462 assert(I.getType() == A->getType());
5463
5464 Value *AShadow = getShadow(V: A);
5465 Value *XShadow = getShadow(V: X);
5466 Value *BZeroShadow = getCleanShadow(V: B);
5467
5468 Value *AShadowXShadow = IRB.CreateIntrinsic(
5469 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {XShadow, AShadow, BZeroShadow});
5470 Value *AShadowX = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5471 Args: {X, AShadow, BZeroShadow});
5472 Value *XShadowA = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5473 Args: {XShadow, A, BZeroShadow});
5474
5475 unsigned NumElements = cast<FixedVectorType>(Val: I.getType())->getNumElements();
5476 Value *BShadow = getShadow(V: B);
5477 Value *BBroadcastShadow = getCleanShadow(V: AShadow);
5478 // There is no LLVM IR intrinsic for _mm512_set1_epi8.
5479 // This loop generates a lot of LLVM IR, which we expect that CodeGen will
5480 // lower appropriately (e.g., VPBROADCASTB).
5481 // Besides, b is often a constant, in which case it is fully initialized.
5482 for (unsigned i = 0; i < NumElements; i++)
5483 BBroadcastShadow = IRB.CreateInsertElement(Vec: BBroadcastShadow, NewElt: BShadow, Idx: i);
5484
5485 setShadow(V: &I, SV: IRB.CreateOr(
5486 Ops: {AShadowXShadow, AShadowX, XShadowA, BBroadcastShadow}));
5487 setOriginForNaryOp(I);
5488 }
5489
5490 // Handle Arm NEON vector load intrinsics (vld*).
5491 //
5492 // The WithLane instructions (ld[234]lane) are similar to:
5493 // call {<4 x i32>, <4 x i32>, <4 x i32>}
5494 // @llvm.aarch64.neon.ld3lane.v4i32.p0
5495 // (<4 x i32> %L1, <4 x i32> %L2, <4 x i32> %L3, i64 %lane, ptr
5496 // %A)
5497 //
5498 // The non-WithLane instructions (ld[234], ld1x[234], ld[234]r) are similar
5499 // to:
5500 // call {<8 x i8>, <8 x i8>} @llvm.aarch64.neon.ld2.v8i8.p0(ptr %A)
5501 void handleNEONVectorLoad(IntrinsicInst &I, bool WithLane) {
5502 unsigned int numArgs = I.arg_size();
5503
5504 // Return type is a struct of vectors of integers or floating-point
5505 assert(I.getType()->isStructTy());
5506 [[maybe_unused]] StructType *RetTy = cast<StructType>(Val: I.getType());
5507 assert(RetTy->getNumElements() > 0);
5508 assert(RetTy->getElementType(0)->isIntOrIntVectorTy() ||
5509 RetTy->getElementType(0)->isFPOrFPVectorTy());
5510 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5511 assert(RetTy->getElementType(i) == RetTy->getElementType(0));
5512
5513 if (WithLane) {
5514 // 2, 3 or 4 vectors, plus lane number, plus input pointer
5515 assert(4 <= numArgs && numArgs <= 6);
5516
5517 // Return type is a struct of the input vectors
5518 assert(RetTy->getNumElements() + 2 == numArgs);
5519 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5520 assert(I.getArgOperand(i)->getType() == RetTy->getElementType(0));
5521 } else {
5522 assert(numArgs == 1);
5523 }
5524
5525 IRBuilder<> IRB(&I);
5526
5527 SmallVector<Value *, 6> ShadowArgs;
5528 if (WithLane) {
5529 for (unsigned int i = 0; i < numArgs - 2; i++)
5530 ShadowArgs.push_back(Elt: getShadow(V: I.getArgOperand(i)));
5531
5532 // Lane number, passed verbatim
5533 Value *LaneNumber = I.getArgOperand(i: numArgs - 2);
5534 ShadowArgs.push_back(Elt: LaneNumber);
5535
5536 // TODO: blend shadow of lane number into output shadow?
5537 insertCheckShadowOf(Val: LaneNumber, OrigIns: &I);
5538 }
5539
5540 Value *Src = I.getArgOperand(i: numArgs - 1);
5541 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
5542
5543 Type *SrcShadowTy = getShadowTy(V: Src);
5544 auto [SrcShadowPtr, SrcOriginPtr] =
5545 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment: Align(1), /*isStore*/ false);
5546 ShadowArgs.push_back(Elt: SrcShadowPtr);
5547
5548 // The NEON vector load instructions handled by this function all have
5549 // integer variants. It is easier to use those rather than trying to cast
5550 // a struct of vectors of floats into a struct of vectors of integers.
5551 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5552 RetTy: getShadowTy(V: &I), ID: I.getIntrinsicID(), Args: ShadowArgs);
5553 setShadow(V: &I, SV: CI);
5554
5555 if (!MS.TrackOrigins)
5556 return;
5557
5558 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
5559 setOrigin(V: &I, Origin: PtrSrcOrigin);
5560 }
5561
5562 /// Handle Arm NEON vector store intrinsics (vst{2,3,4}, vst1x_{2,3,4},
5563 /// and vst{2,3,4}lane).
5564 ///
5565 /// Arm NEON vector store intrinsics have the output address (pointer) as the
5566 /// last argument, with the initial arguments being the inputs (and lane
5567 /// number for vst{2,3,4}lane). They return void.
5568 ///
5569 /// - st4 interleaves the output e.g., st4 (inA, inB, inC, inD, outP) writes
5570 /// abcdabcdabcdabcd... into *outP
5571 /// - st1_x4 is non-interleaved e.g., st1_x4 (inA, inB, inC, inD, outP)
5572 /// writes aaaa...bbbb...cccc...dddd... into *outP
5573 /// - st4lane has arguments of (inA, inB, inC, inD, lane, outP)
5574 /// These instructions can all be instrumented with essentially the same
5575 /// MSan logic, simply by applying the corresponding intrinsic to the shadow.
5576 void handleNEONVectorStoreIntrinsic(IntrinsicInst &I, bool useLane) {
5577 IRBuilder<> IRB(&I);
5578
5579 // Don't use getNumOperands() because it includes the callee
5580 int numArgOperands = I.arg_size();
5581
5582 // The last arg operand is the output (pointer)
5583 assert(numArgOperands >= 1);
5584 Value *Addr = I.getArgOperand(i: numArgOperands - 1);
5585 assert(Addr->getType()->isPointerTy());
5586 int skipTrailingOperands = 1;
5587
5588 if (ClCheckAccessAddress)
5589 insertCheckShadowOf(Val: Addr, OrigIns: &I);
5590
5591 // Second-last operand is the lane number (for vst{2,3,4}lane)
5592 if (useLane) {
5593 skipTrailingOperands++;
5594 assert(numArgOperands >= static_cast<int>(skipTrailingOperands));
5595 assert(isa<IntegerType>(
5596 I.getArgOperand(numArgOperands - skipTrailingOperands)->getType()));
5597 }
5598
5599 SmallVector<Value *, 8> ShadowArgs;
5600 // All the initial operands are the inputs
5601 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++) {
5602 assert(isa<FixedVectorType>(I.getArgOperand(i)->getType()));
5603 Value *Shadow = getShadow(I: &I, i);
5604 ShadowArgs.append(NumInputs: 1, Elt: Shadow);
5605 }
5606
5607 // MSan's GetShadowTy assumes the LHS is the type we want the shadow for
5608 // e.g., for:
5609 // [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
5610 // we know the type of the output (and its shadow) is <16 x i8>.
5611 //
5612 // Arm NEON VST is unusual because the last argument is the output address:
5613 // define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) {
5614 // call void @llvm.aarch64.neon.st2.v16i8.p0
5615 // (<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
5616 // and we have no type information about P's operand. We must manually
5617 // compute the type (<16 x i8> x 2).
5618 FixedVectorType *OutputVectorTy = FixedVectorType::get(
5619 ElementType: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getElementType(),
5620 NumElts: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements() *
5621 (numArgOperands - skipTrailingOperands));
5622 Type *OutputShadowTy = getShadowTy(OrigTy: OutputVectorTy);
5623
5624 if (useLane)
5625 ShadowArgs.append(NumInputs: 1,
5626 Elt: I.getArgOperand(i: numArgOperands - skipTrailingOperands));
5627
5628 Value *OutputShadowPtr, *OutputOriginPtr;
5629 // AArch64 NEON does not need alignment (unless OS requires it)
5630 std::tie(args&: OutputShadowPtr, args&: OutputOriginPtr) = getShadowOriginPtr(
5631 Addr, IRB, ShadowTy: OutputShadowTy, Alignment: Align(1), /*isStore*/ true);
5632 ShadowArgs.append(NumInputs: 1, Elt: OutputShadowPtr);
5633
5634 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5635 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
5636 setShadow(V: &I, SV: CI);
5637
5638 if (MS.TrackOrigins) {
5639 // TODO: if we modelled the vst* instruction more precisely, we could
5640 // more accurately track the origins (e.g., if both inputs are
5641 // uninitialized for vst2, we currently blame the second input, even
5642 // though part of the output depends only on the first input).
5643 //
5644 // This is particularly imprecise for vst{2,3,4}lane, since only one
5645 // lane of each input is actually copied to the output.
5646 OriginCombiner OC(this, IRB);
5647 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++)
5648 OC.Add(V: I.getArgOperand(i));
5649
5650 const DataLayout &DL = F.getDataLayout();
5651 OC.DoneAndStoreOrigin(TS: DL.getTypeStoreSize(Ty: OutputVectorTy),
5652 OriginPtr: OutputOriginPtr);
5653 }
5654 }
5655
5656 // Integer matrix multiplication:
5657 // - <4 x i32> @llvm.aarch64.neon.{s,u,us}mmla.v4i32.v16i8
5658 // (<4 x i32> %R, <16 x i8> %A, <16 x i8> %B)
5659 // - <4 x i32> is a 2x2 matrix
5660 // - <16 x i8> %A and %B are 2x8 and 8x2 matrices respectively
5661 //
5662 // Floating-point matrix multiplication:
5663 // - <4 x float> @llvm.aarch64.neon.bfmmla
5664 // (<4 x float> %R, <8 x bfloat> %A, <8 x bfloat> %B)
5665 // - <4 x float> is a 2x2 matrix
5666 // - <8 x bfloat> %A and %B are 2x4 and 4x2 matrices respectively
5667 //
5668 // The general shadow propagation approach is:
5669 // 1) get the shadows of the input matrices %A and %B
5670 // 2) map each shadow value to 0x1 if the corresponding value is fully
5671 // initialized, and 0x0 otherwise
5672 // 3) perform a matrix multiplication on the shadows of %A and %B [*].
5673 // The output will be a 2x2 matrix. For each element, a value of 0x8
5674 // (for {s,u,us}mmla) or 0x4 (for bfmmla) means all the corresponding
5675 // inputs were clean; if so, set the shadow to zero, otherwise set to -1.
5676 // 4) blend in the shadow of %R
5677 //
5678 // [*] Since shadows are integral, the obvious approach is to always apply
5679 // ummla to the shadows. Unfortunately, Armv8.2+bf16 supports bfmmla,
5680 // but not ummla. Thus, for bfmmla, our instrumentation reuses bfmmla.
5681 //
5682 // TODO: consider allowing multiplication of zero with an uninitialized value
5683 // to result in an initialized value.
5684 void handleNEONMatrixMultiply(IntrinsicInst &I) {
5685 IRBuilder<> IRB(&I);
5686
5687 assert(I.arg_size() == 3);
5688 Value *R = I.getArgOperand(i: 0);
5689 Value *A = I.getArgOperand(i: 1);
5690 Value *B = I.getArgOperand(i: 2);
5691
5692 assert(I.getType() == R->getType());
5693
5694 assert(isa<FixedVectorType>(R->getType()));
5695 assert(isa<FixedVectorType>(A->getType()));
5696 assert(isa<FixedVectorType>(B->getType()));
5697
5698 FixedVectorType *RTy = cast<FixedVectorType>(Val: R->getType());
5699 FixedVectorType *ATy = cast<FixedVectorType>(Val: A->getType());
5700 FixedVectorType *BTy = cast<FixedVectorType>(Val: B->getType());
5701 assert(ATy->getElementType() == BTy->getElementType());
5702
5703 if (RTy->getElementType()->isIntegerTy()) {
5704 // <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8
5705 // (<4 x i32> %R, <16 x i8> %X, <16 x i8> %Y)
5706 assert(RTy == FixedVectorType::get(IntegerType::get(*MS.C, 32), 4));
5707 assert(ATy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5708 assert(BTy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5709 } else {
5710 // <4 x float> @llvm.aarch64.neon.bfmmla
5711 // (<4 x float> %R, <8 x bfloat> %X, <8 x bfloat> %Y)
5712 assert(RTy == FixedVectorType::get(Type::getFloatTy(*MS.C), 4));
5713 assert(ATy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5714 assert(BTy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5715 }
5716
5717 Value *ShadowR = getShadow(I: &I, i: 0);
5718 Value *ShadowA = getShadow(I: &I, i: 1);
5719 Value *ShadowB = getShadow(I: &I, i: 2);
5720
5721 Value *ShadowAB;
5722 Value *FullyInit;
5723
5724 if (RTy->getElementType()->isIntegerTy()) {
5725 // If the value is fully initialized, the shadow will be 000...001.
5726 // Otherwise, the shadow will be all zero.
5727 // (This is the opposite of how we typically handle shadows.)
5728 ShadowA = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5729 DestTy: getShadowTy(OrigTy: ATy));
5730 ShadowB = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5731 DestTy: getShadowTy(OrigTy: BTy));
5732 // TODO: the CreateSelect approach used below for floating-point is more
5733 // generic than CreateZExt. Investigate whether it is worthwhile
5734 // unifying the two approaches.
5735
5736 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_ummla,
5737 Args: {getCleanShadow(OrigTy: RTy), ShadowA, ShadowB});
5738
5739 // ummla multiplies a 2x8 matrix with an 8x2 matrix. If all entries of the
5740 // input matrices are equal to 0x1, all entries of the output matrix will
5741 // be 0x8.
5742 FullyInit = ConstantVector::getSplat(
5743 EC: RTy->getElementCount(), Elt: ConstantInt::get(Ty: RTy->getElementType(), V: 0x8));
5744
5745 ShadowAB = IRB.CreateICmpNE(LHS: ShadowAB, RHS: FullyInit);
5746 } else {
5747 Constant *ABZeros = ConstantVector::getSplat(
5748 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 0));
5749 Constant *ABOnes = ConstantVector::getSplat(
5750 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 1));
5751
5752 // As per the integer case, if the shadow is clean, we store 0x1,
5753 // otherwise we store 0x0 (the opposite of usual shadow arithmetic).
5754 ShadowA = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5755 True: ABOnes, False: ABZeros);
5756 ShadowB = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5757 True: ABOnes, False: ABZeros);
5758
5759 Constant *RZeros = ConstantVector::getSplat(
5760 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 0));
5761
5762 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_bfmmla,
5763 Args: {RZeros, ShadowA, ShadowB});
5764
5765 // bfmmla multiplies a 2x4 matrix with an 4x2 matrix. If all entries of
5766 // the input matrices are equal to 0x1, all entries of the output matrix
5767 // will be 4.0. (To avoid floating-point error, we check if each entry
5768 // < 3.5.)
5769 FullyInit = ConstantVector::getSplat(
5770 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 3.5));
5771
5772 // FCmpULT: "yields true if either operand is a QNAN or op1 is less than"
5773 // op2"
5774 ShadowAB = IRB.CreateFCmpULT(LHS: ShadowAB, RHS: FullyInit);
5775 }
5776
5777 ShadowR = IRB.CreateICmpNE(LHS: ShadowR, RHS: getCleanShadow(OrigTy: RTy));
5778 ShadowR = IRB.CreateOr(LHS: ShadowAB, RHS: ShadowR);
5779
5780 setShadow(V: &I, SV: IRB.CreateSExt(V: ShadowR, DestTy: getShadowTy(OrigTy: RTy)));
5781
5782 setOriginForNaryOp(I);
5783 }
5784
5785 /// Handle intrinsics by applying the intrinsic to the shadows.
5786 ///
5787 /// For example, this can be applied to the Arm NEON vector table intrinsics
5788 /// (tbl{1,2,3,4}).
5789 ///
5790 /// Typically, shadowIntrinsicID will be specified by the caller to be
5791 /// I.getIntrinsicID(), but the caller can choose to replace it with another
5792 /// intrinsic of the same type.
5793 ///
5794 /// The trailing arguments are passed verbatim to the intrinsic, though any
5795 /// uninitialized trailing arguments can also taint the shadow e.g., for an
5796 /// intrinsic with one trailing verbatim argument:
5797 /// out = intrinsic(var1, var2, opType)
5798 /// we compute:
5799 /// shadow[out] =
5800 /// intrinsic(shadow[var1], shadow[var2], opType) | shadow[opType]
5801 ///
5802 /// If an intrinsic is called with floating-point arguments, we will
5803 /// typically cast the shadows to floating-point, apply the intrinsic [*],
5804 /// then cast the result back to integer/shadow.
5805 ///
5806 /// In cases where we know the intrinsic is compatible with integer
5807 /// arguments, 'forceIntegerIntrinsic' will apply the integer variant, even
5808 /// if the arguments are floating-point, thus avoiding unnecessary casts
5809 /// e.g., if I is:
5810 /// <16 x float> @llvm.x86.avx512.mask.compress
5811 /// (<16 x float>, <16 x float>, <16 x i1> %mask)
5812 /// we would prefer to compute the shadows using:
5813 /// <16 x i32> @llvm.x86.avx512.mask.compress
5814 /// (<16 x i32>, <16 x i32>, <16 x i1> %mask)
5815 ///
5816 /// [*] CAUTION: this assumes that the intrinsic will handle arbitrary
5817 /// bit-patterns (for example, if the intrinsic accepts floats
5818 /// for var1, we require that it doesn't care if inputs are
5819 /// NaNs).
5820 ///
5821 /// The origin is approximated using setOriginForNaryOp.
5822 void handleIntrinsicByApplyingToShadow(IntrinsicInst &I,
5823 Intrinsic::ID shadowIntrinsicID,
5824 unsigned int trailingVerbatimArgs,
5825 bool forceIntegerIntrinsic) {
5826 IRBuilder<> IRB(&I);
5827
5828 assert(trailingVerbatimArgs < I.arg_size());
5829
5830 SmallVector<Value *, 8> ShadowArgs;
5831 // Don't use getNumOperands() because it includes the callee
5832 for (unsigned int i = 0; i < I.arg_size() - trailingVerbatimArgs; i++) {
5833 Value *Shadow = getShadow(I: &I, i);
5834
5835 if (forceIntegerIntrinsic)
5836 ShadowArgs.push_back(Elt: Shadow);
5837 else
5838 ShadowArgs.push_back(
5839 Elt: IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i)->getType()));
5840 }
5841
5842 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5843 i++) {
5844 Value *Arg = I.getArgOperand(i);
5845 if (forceIntegerIntrinsic)
5846 assert(Arg->getType()->isIntOrIntVectorTy());
5847 ShadowArgs.push_back(Elt: Arg);
5848 }
5849
5850 Value *CombinedShadow;
5851 if (forceIntegerIntrinsic) {
5852 CombinedShadow =
5853 IRB.CreateIntrinsic(RetTy: getShadowTy(V: &I), ID: shadowIntrinsicID, Args: ShadowArgs);
5854 } else {
5855 Value *CI =
5856 IRB.CreateIntrinsic(RetTy: I.getType(), ID: shadowIntrinsicID, Args: ShadowArgs);
5857 CombinedShadow = IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I));
5858 }
5859
5860 // Combine the computed shadow with the shadow of trailing args
5861 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5862 i++) {
5863 Value *Shadow =
5864 CreateShadowCast(IRB, V: getShadow(I: &I, i), dstTy: CombinedShadow->getType());
5865 CombinedShadow = IRB.CreateOr(LHS: Shadow, RHS: CombinedShadow, Name: "_msprop");
5866 }
5867
5868 setShadow(V: &I, SV: CombinedShadow);
5869
5870 setOriginForNaryOp(I);
5871 }
5872
5873 // Approximation only
5874 //
5875 // e.g., <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
5876 void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
5877 assert(I.arg_size() == 2);
5878
5879 handleShadowOr(I);
5880 }
5881
5882 bool maybeHandleCrossPlatformIntrinsic(IntrinsicInst &I) {
5883 switch (I.getIntrinsicID()) {
5884 case Intrinsic::uadd_with_overflow:
5885 case Intrinsic::sadd_with_overflow:
5886 case Intrinsic::usub_with_overflow:
5887 case Intrinsic::ssub_with_overflow:
5888 case Intrinsic::umul_with_overflow:
5889 case Intrinsic::smul_with_overflow:
5890 handleArithmeticWithOverflow(I);
5891 break;
5892 case Intrinsic::modf:
5893 case Intrinsic::sincos:
5894 case Intrinsic::sincospi:
5895 handleModfOrSincos(I);
5896 break;
5897 case Intrinsic::abs:
5898 handleAbsIntrinsic(I);
5899 break;
5900 case Intrinsic::bitreverse:
5901 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
5902 /*trailingVerbatimArgs=*/0,
5903 /*forceIntegerIntrinsic=*/false);
5904 break;
5905 case Intrinsic::is_fpclass:
5906 handleIsFpClass(I);
5907 break;
5908 case Intrinsic::lifetime_start:
5909 handleLifetimeStart(I);
5910 break;
5911 case Intrinsic::launder_invariant_group:
5912 case Intrinsic::strip_invariant_group:
5913 handleInvariantGroup(I);
5914 break;
5915 case Intrinsic::bswap:
5916 handleBswap(I);
5917 break;
5918 case Intrinsic::ctlz:
5919 case Intrinsic::cttz:
5920 handleCountLeadingTrailingZeros(I);
5921 break;
5922 case Intrinsic::masked_compressstore:
5923 handleMaskedCompressStore(I);
5924 break;
5925 case Intrinsic::masked_expandload:
5926 handleMaskedExpandLoad(I);
5927 break;
5928 case Intrinsic::masked_gather:
5929 handleMaskedGather(I);
5930 break;
5931 case Intrinsic::masked_scatter:
5932 handleMaskedScatter(I);
5933 break;
5934 case Intrinsic::masked_store:
5935 handleMaskedStore(I);
5936 break;
5937 case Intrinsic::masked_load:
5938 handleMaskedLoad(I);
5939 break;
5940 case Intrinsic::vector_reduce_and:
5941 handleVectorReduceAndIntrinsic(I);
5942 break;
5943 case Intrinsic::vector_reduce_or:
5944 handleVectorReduceOrIntrinsic(I);
5945 break;
5946
5947 case Intrinsic::vector_reduce_add:
5948 case Intrinsic::vector_reduce_xor:
5949 case Intrinsic::vector_reduce_mul:
5950 // Signed/Unsigned Min/Max
5951 // TODO: handling similarly to AND/OR may be more precise.
5952 case Intrinsic::vector_reduce_smax:
5953 case Intrinsic::vector_reduce_smin:
5954 case Intrinsic::vector_reduce_umax:
5955 case Intrinsic::vector_reduce_umin:
5956 // TODO: this has no false positives, but arguably we should check that all
5957 // the bits are initialized.
5958 case Intrinsic::vector_reduce_fmax:
5959 case Intrinsic::vector_reduce_fmin:
5960 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/false);
5961 break;
5962
5963 case Intrinsic::vector_reduce_fadd:
5964 case Intrinsic::vector_reduce_fmul:
5965 handleVectorReduceWithStarterIntrinsic(I);
5966 break;
5967
5968 case Intrinsic::scmp:
5969 case Intrinsic::ucmp: {
5970 handleShadowOr(I);
5971 break;
5972 }
5973
5974 case Intrinsic::fshl:
5975 case Intrinsic::fshr:
5976 handleFunnelShift(I);
5977 break;
5978
5979 case Intrinsic::pdep:
5980 case Intrinsic::pext:
5981 handleGenericBitManipulation(I);
5982 break;
5983
5984 case Intrinsic::is_constant:
5985 // The result of llvm.is.constant() is always defined.
5986 setShadow(V: &I, SV: getCleanShadow(V: &I));
5987 setOrigin(V: &I, Origin: getCleanOrigin());
5988 break;
5989
5990 // The non-saturating versions are handled by visitFPTo[US]IInst().
5991 //
5992 // N.B. some platform-specific intrinsics, such as AArch64 fcvtz[us], are
5993 // lowered to these cross-platform intrinsics.
5994 case Intrinsic::fptosi_sat:
5995 case Intrinsic::fptoui_sat:
5996 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
5997 break;
5998
5999 default:
6000 return false;
6001 }
6002
6003 return true;
6004 }
6005
6006 bool maybeHandleX86SIMDIntrinsic(IntrinsicInst &I) {
6007 switch (I.getIntrinsicID()) {
6008 case Intrinsic::x86_sse_stmxcsr:
6009 handleStmxcsr(I);
6010 break;
6011 case Intrinsic::x86_sse_ldmxcsr:
6012 handleLdmxcsr(I);
6013 break;
6014
6015 // Convert Scalar Double Precision Floating-Point Value
6016 // to Unsigned Doubleword Integer
6017 // etc.
6018 case Intrinsic::x86_avx512_vcvtsd2usi64:
6019 case Intrinsic::x86_avx512_vcvtsd2usi32:
6020 case Intrinsic::x86_avx512_vcvtss2usi64:
6021 case Intrinsic::x86_avx512_vcvtss2usi32:
6022 case Intrinsic::x86_avx512_cvttss2usi64:
6023 case Intrinsic::x86_avx512_cvttss2usi:
6024 case Intrinsic::x86_avx512_cvttsd2usi64:
6025 case Intrinsic::x86_avx512_cvttsd2usi:
6026 case Intrinsic::x86_avx512_cvtusi2ss:
6027 case Intrinsic::x86_avx512_cvtusi642sd:
6028 case Intrinsic::x86_avx512_cvtusi642ss:
6029 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1, HasRoundingMode: true);
6030 break;
6031 case Intrinsic::x86_sse2_cvtsd2si64:
6032 case Intrinsic::x86_sse2_cvtsd2si:
6033 case Intrinsic::x86_sse2_cvtsd2ss:
6034 case Intrinsic::x86_sse2_cvttsd2si64:
6035 case Intrinsic::x86_sse2_cvttsd2si:
6036 case Intrinsic::x86_sse_cvtss2si64:
6037 case Intrinsic::x86_sse_cvtss2si:
6038 case Intrinsic::x86_sse_cvttss2si64:
6039 case Intrinsic::x86_sse_cvttss2si:
6040 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1);
6041 break;
6042 case Intrinsic::x86_sse_cvtps2pi:
6043 case Intrinsic::x86_sse_cvttps2pi:
6044 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 2);
6045 break;
6046
6047 // TODO:
6048 // <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double>)
6049 // <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64>)
6050 // <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, <1 x i64>)
6051
6052 case Intrinsic::x86_vcvtps2ph_128:
6053 case Intrinsic::x86_vcvtps2ph_256: {
6054 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/true);
6055 break;
6056 }
6057
6058 // Convert Packed Single Precision Floating-Point Values
6059 // to Packed Signed Doubleword Integer Values
6060 //
6061 // <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
6062 // (<16 x float>, <16 x i32>, i16, i32)
6063 case Intrinsic::x86_avx512_mask_cvtps2dq_512:
6064 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/false);
6065 break;
6066
6067 // Convert Packed Double Precision Floating-Point Values
6068 // to Packed Single Precision Floating-Point Values
6069 case Intrinsic::x86_sse2_cvtpd2ps:
6070 case Intrinsic::x86_sse2_cvtps2dq:
6071 case Intrinsic::x86_sse2_cvtpd2dq:
6072 case Intrinsic::x86_sse2_cvttps2dq:
6073 case Intrinsic::x86_sse2_cvttpd2dq:
6074 case Intrinsic::x86_avx_cvt_pd2_ps_256:
6075 case Intrinsic::x86_avx_cvt_ps2dq_256:
6076 case Intrinsic::x86_avx_cvt_pd2dq_256:
6077 case Intrinsic::x86_avx_cvtt_ps2dq_256:
6078 case Intrinsic::x86_avx_cvtt_pd2dq_256: {
6079 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/false);
6080 break;
6081 }
6082
6083 // Convert Single-Precision FP Value to 16-bit FP Value
6084 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
6085 // (<16 x float>, i32, <16 x i16>, i16)
6086 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
6087 // (<4 x float>, i32, <8 x i16>, i8)
6088 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256
6089 // (<8 x float>, i32, <8 x i16>, i8)
6090 case Intrinsic::x86_avx512_mask_vcvtps2ph_512:
6091 case Intrinsic::x86_avx512_mask_vcvtps2ph_256:
6092 case Intrinsic::x86_avx512_mask_vcvtps2ph_128:
6093 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/true);
6094 break;
6095
6096 // Shift Packed Data (Left Logical, Right Arithmetic, Right Logical)
6097 case Intrinsic::x86_avx512_psll_w_512:
6098 case Intrinsic::x86_avx512_psll_d_512:
6099 case Intrinsic::x86_avx512_psll_q_512:
6100 case Intrinsic::x86_avx512_pslli_w_512:
6101 case Intrinsic::x86_avx512_pslli_d_512:
6102 case Intrinsic::x86_avx512_pslli_q_512:
6103 case Intrinsic::x86_avx512_psrl_w_512:
6104 case Intrinsic::x86_avx512_psrl_d_512:
6105 case Intrinsic::x86_avx512_psrl_q_512:
6106 case Intrinsic::x86_avx512_psra_w_512:
6107 case Intrinsic::x86_avx512_psra_d_512:
6108 case Intrinsic::x86_avx512_psra_q_512:
6109 case Intrinsic::x86_avx512_psrli_w_512:
6110 case Intrinsic::x86_avx512_psrli_d_512:
6111 case Intrinsic::x86_avx512_psrli_q_512:
6112 case Intrinsic::x86_avx512_psrai_w_512:
6113 case Intrinsic::x86_avx512_psrai_d_512:
6114 case Intrinsic::x86_avx512_psrai_q_512:
6115 case Intrinsic::x86_avx512_psra_q_256:
6116 case Intrinsic::x86_avx512_psra_q_128:
6117 case Intrinsic::x86_avx512_psrai_q_256:
6118 case Intrinsic::x86_avx512_psrai_q_128:
6119 case Intrinsic::x86_avx2_psll_w:
6120 case Intrinsic::x86_avx2_psll_d:
6121 case Intrinsic::x86_avx2_psll_q:
6122 case Intrinsic::x86_avx2_pslli_w:
6123 case Intrinsic::x86_avx2_pslli_d:
6124 case Intrinsic::x86_avx2_pslli_q:
6125 case Intrinsic::x86_avx2_psrl_w:
6126 case Intrinsic::x86_avx2_psrl_d:
6127 case Intrinsic::x86_avx2_psrl_q:
6128 case Intrinsic::x86_avx2_psra_w:
6129 case Intrinsic::x86_avx2_psra_d:
6130 case Intrinsic::x86_avx2_psrli_w:
6131 case Intrinsic::x86_avx2_psrli_d:
6132 case Intrinsic::x86_avx2_psrli_q:
6133 case Intrinsic::x86_avx2_psrai_w:
6134 case Intrinsic::x86_avx2_psrai_d:
6135 case Intrinsic::x86_sse2_psll_w:
6136 case Intrinsic::x86_sse2_psll_d:
6137 case Intrinsic::x86_sse2_psll_q:
6138 case Intrinsic::x86_sse2_pslli_w:
6139 case Intrinsic::x86_sse2_pslli_d:
6140 case Intrinsic::x86_sse2_pslli_q:
6141 case Intrinsic::x86_sse2_psrl_w:
6142 case Intrinsic::x86_sse2_psrl_d:
6143 case Intrinsic::x86_sse2_psrl_q:
6144 case Intrinsic::x86_sse2_psra_w:
6145 case Intrinsic::x86_sse2_psra_d:
6146 case Intrinsic::x86_sse2_psrli_w:
6147 case Intrinsic::x86_sse2_psrli_d:
6148 case Intrinsic::x86_sse2_psrli_q:
6149 case Intrinsic::x86_sse2_psrai_w:
6150 case Intrinsic::x86_sse2_psrai_d:
6151 case Intrinsic::x86_mmx_psll_w:
6152 case Intrinsic::x86_mmx_psll_d:
6153 case Intrinsic::x86_mmx_psll_q:
6154 case Intrinsic::x86_mmx_pslli_w:
6155 case Intrinsic::x86_mmx_pslli_d:
6156 case Intrinsic::x86_mmx_pslli_q:
6157 case Intrinsic::x86_mmx_psrl_w:
6158 case Intrinsic::x86_mmx_psrl_d:
6159 case Intrinsic::x86_mmx_psrl_q:
6160 case Intrinsic::x86_mmx_psra_w:
6161 case Intrinsic::x86_mmx_psra_d:
6162 case Intrinsic::x86_mmx_psrli_w:
6163 case Intrinsic::x86_mmx_psrli_d:
6164 case Intrinsic::x86_mmx_psrli_q:
6165 case Intrinsic::x86_mmx_psrai_w:
6166 case Intrinsic::x86_mmx_psrai_d:
6167 handleVectorShiftIntrinsic(I, /* Variable */ false);
6168 break;
6169 case Intrinsic::x86_avx2_psllv_d:
6170 case Intrinsic::x86_avx2_psllv_d_256:
6171 case Intrinsic::x86_avx512_psllv_d_512:
6172 case Intrinsic::x86_avx2_psllv_q:
6173 case Intrinsic::x86_avx2_psllv_q_256:
6174 case Intrinsic::x86_avx512_psllv_q_512:
6175 case Intrinsic::x86_avx2_psrlv_d:
6176 case Intrinsic::x86_avx2_psrlv_d_256:
6177 case Intrinsic::x86_avx512_psrlv_d_512:
6178 case Intrinsic::x86_avx2_psrlv_q:
6179 case Intrinsic::x86_avx2_psrlv_q_256:
6180 case Intrinsic::x86_avx512_psrlv_q_512:
6181 case Intrinsic::x86_avx2_psrav_d:
6182 case Intrinsic::x86_avx2_psrav_d_256:
6183 case Intrinsic::x86_avx512_psrav_d_512:
6184 case Intrinsic::x86_avx512_psrav_q_128:
6185 case Intrinsic::x86_avx512_psrav_q_256:
6186 case Intrinsic::x86_avx512_psrav_q_512:
6187 handleVectorShiftIntrinsic(I, /* Variable */ true);
6188 break;
6189
6190 // Pack with Signed/Unsigned Saturation
6191 case Intrinsic::x86_sse2_packsswb_128:
6192 case Intrinsic::x86_sse2_packssdw_128:
6193 case Intrinsic::x86_sse2_packuswb_128:
6194 case Intrinsic::x86_sse41_packusdw:
6195 case Intrinsic::x86_avx2_packsswb:
6196 case Intrinsic::x86_avx2_packssdw:
6197 case Intrinsic::x86_avx2_packuswb:
6198 case Intrinsic::x86_avx2_packusdw:
6199 // e.g., <64 x i8> @llvm.x86.avx512.packsswb.512
6200 // (<32 x i16> %a, <32 x i16> %b)
6201 // <32 x i16> @llvm.x86.avx512.packssdw.512
6202 // (<16 x i32> %a, <16 x i32> %b)
6203 // Note: AVX512 masked variants are auto-upgraded by LLVM.
6204 case Intrinsic::x86_avx512_packsswb_512:
6205 case Intrinsic::x86_avx512_packssdw_512:
6206 case Intrinsic::x86_avx512_packuswb_512:
6207 case Intrinsic::x86_avx512_packusdw_512:
6208 handleVectorPackIntrinsic(I);
6209 break;
6210
6211 case Intrinsic::x86_sse41_pblendvb:
6212 case Intrinsic::x86_sse41_blendvpd:
6213 case Intrinsic::x86_sse41_blendvps:
6214 case Intrinsic::x86_avx_blendv_pd_256:
6215 case Intrinsic::x86_avx_blendv_ps_256:
6216 case Intrinsic::x86_avx2_pblendvb:
6217 handleBlendvIntrinsic(I);
6218 break;
6219
6220 case Intrinsic::x86_avx_dp_ps_256:
6221 case Intrinsic::x86_sse41_dppd:
6222 case Intrinsic::x86_sse41_dpps:
6223 handleDppIntrinsic(I);
6224 break;
6225
6226 case Intrinsic::x86_mmx_packsswb:
6227 case Intrinsic::x86_mmx_packuswb:
6228 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 16);
6229 break;
6230
6231 case Intrinsic::x86_mmx_packssdw:
6232 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 32);
6233 break;
6234
6235 case Intrinsic::x86_mmx_psad_bw:
6236 handleVectorSadIntrinsic(I, IsMMX: true);
6237 break;
6238 case Intrinsic::x86_sse2_psad_bw:
6239 case Intrinsic::x86_avx2_psad_bw:
6240 handleVectorSadIntrinsic(I);
6241 break;
6242
6243 // Multiply and Add Packed Words
6244 // < 4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
6245 // < 8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
6246 // <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
6247 //
6248 // Multiply and Add Packed Signed and Unsigned Bytes
6249 // < 8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
6250 // <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
6251 // <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
6252 //
6253 // These intrinsics are auto-upgraded into non-masked forms:
6254 // < 4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128
6255 // (<8 x i16>, <8 x i16>, <4 x i32>, i8)
6256 // < 8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256
6257 // (<16 x i16>, <16 x i16>, <8 x i32>, i8)
6258 // <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512
6259 // (<32 x i16>, <32 x i16>, <16 x i32>, i16)
6260 // < 8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128
6261 // (<16 x i8>, <16 x i8>, <8 x i16>, i8)
6262 // <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256
6263 // (<32 x i8>, <32 x i8>, <16 x i16>, i16)
6264 // <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512
6265 // (<64 x i8>, <64 x i8>, <32 x i16>, i32)
6266 case Intrinsic::x86_sse2_pmadd_wd:
6267 case Intrinsic::x86_avx2_pmadd_wd:
6268 case Intrinsic::x86_avx512_pmaddw_d_512:
6269 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
6270 case Intrinsic::x86_avx2_pmadd_ub_sw:
6271 case Intrinsic::x86_avx512_pmaddubs_w_512:
6272 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6273 /*ZeroPurifies=*/true,
6274 /*EltSizeInBits=*/0,
6275 /*Lanes=*/kBothLanes);
6276 break;
6277
6278 // <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>)
6279 case Intrinsic::x86_ssse3_pmadd_ub_sw:
6280 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6281 /*ZeroPurifies=*/true,
6282 /*EltSizeInBits=*/8,
6283 /*Lanes=*/kBothLanes);
6284 break;
6285
6286 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>)
6287 case Intrinsic::x86_mmx_pmadd_wd:
6288 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6289 /*ZeroPurifies=*/true,
6290 /*EltSizeInBits=*/16,
6291 /*Lanes=*/kBothLanes);
6292 break;
6293
6294 // BFloat16 multiply-add to single-precision
6295 // <4 x float> llvm.aarch64.neon.bfmlalt
6296 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6297 case Intrinsic::aarch64_neon_bfmlalt:
6298 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6299 /*ZeroPurifies=*/false,
6300 /*EltSizeInBits=*/0,
6301 /*Lanes=*/kOddLanes);
6302 break;
6303
6304 // <4 x float> llvm.aarch64.neon.bfmlalb
6305 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6306 case Intrinsic::aarch64_neon_bfmlalb:
6307 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6308 /*ZeroPurifies=*/false,
6309 /*EltSizeInBits=*/0,
6310 /*Lanes=*/kEvenLanes);
6311 break;
6312
6313 // AVX Vector Neural Network Instructions: bytes
6314 //
6315 // Multiply and Add Signed Bytes
6316 // < 4 x i32> @llvm.x86.avx2.vpdpbssd.128
6317 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6318 // < 8 x i32> @llvm.x86.avx2.vpdpbssd.256
6319 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6320 // <16 x i32> @llvm.x86.avx10.vpdpbssd.512
6321 // (<16 x i32>, <64 x i8>, <64 x i8>)
6322 //
6323 // Multiply and Add Signed Bytes With Saturation
6324 // < 4 x i32> @llvm.x86.avx2.vpdpbssds.128
6325 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6326 // < 8 x i32> @llvm.x86.avx2.vpdpbssds.256
6327 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6328 // <16 x i32> @llvm.x86.avx10.vpdpbssds.512
6329 // (<16 x i32>, <64 x i8>, <64 x i8>)
6330 //
6331 // Multiply and Add Signed and Unsigned Bytes
6332 // < 4 x i32> @llvm.x86.avx2.vpdpbsud.128
6333 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6334 // < 8 x i32> @llvm.x86.avx2.vpdpbsud.256
6335 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6336 // <16 x i32> @llvm.x86.avx10.vpdpbsud.512
6337 // (<16 x i32>, <64 x i8>, <64 x i8>)
6338 //
6339 // Multiply and Add Signed and Unsigned Bytes With Saturation
6340 // < 4 x i32> @llvm.x86.avx2.vpdpbsuds.128
6341 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6342 // < 8 x i32> @llvm.x86.avx2.vpdpbsuds.256
6343 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6344 // <16 x i32> @llvm.x86.avx512.vpdpbusds.512
6345 // (<16 x i32>, <64 x i8>, <64 x i8>)
6346 //
6347 // Multiply and Add Unsigned and Signed Bytes
6348 // < 4 x i32> @llvm.x86.avx512.vpdpbusd.128
6349 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6350 // < 8 x i32> @llvm.x86.avx512.vpdpbusd.256
6351 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6352 // <16 x i32> @llvm.x86.avx512.vpdpbusd.512
6353 // (<16 x i32>, <64 x i8>, <64 x i8>)
6354 //
6355 // Multiply and Add Unsigned and Signed Bytes With Saturation
6356 // < 4 x i32> @llvm.x86.avx512.vpdpbusds.128
6357 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6358 // < 8 x i32> @llvm.x86.avx512.vpdpbusds.256
6359 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6360 // <16 x i32> @llvm.x86.avx10.vpdpbsuds.512
6361 // (<16 x i32>, <64 x i8>, <64 x i8>)
6362 //
6363 // Multiply and Add Unsigned Bytes
6364 // < 4 x i32> @llvm.x86.avx2.vpdpbuud.128
6365 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6366 // < 8 x i32> @llvm.x86.avx2.vpdpbuud.256
6367 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6368 // <16 x i32> @llvm.x86.avx10.vpdpbuud.512
6369 // (<16 x i32>, <64 x i8>, <64 x i8>)
6370 //
6371 // Multiply and Add Unsigned Bytes With Saturation
6372 // < 4 x i32> @llvm.x86.avx2.vpdpbuuds.128
6373 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6374 // < 8 x i32> @llvm.x86.avx2.vpdpbuuds.256
6375 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6376 // <16 x i32> @llvm.x86.avx10.vpdpbuuds.512
6377 // (<16 x i32>, <64 x i8>, <64 x i8>)
6378 //
6379 // These intrinsics are auto-upgraded into non-masked forms:
6380 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128
6381 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6382 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128
6383 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6384 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256
6385 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6386 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256
6387 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6388 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512
6389 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6390 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512
6391 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6392 //
6393 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128
6394 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6395 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128
6396 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6397 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256
6398 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6399 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256
6400 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6401 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512
6402 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6403 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512
6404 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6405 case Intrinsic::x86_avx512_vpdpbusd_128:
6406 case Intrinsic::x86_avx512_vpdpbusd_256:
6407 case Intrinsic::x86_avx512_vpdpbusd_512:
6408 case Intrinsic::x86_avx512_vpdpbusds_128:
6409 case Intrinsic::x86_avx512_vpdpbusds_256:
6410 case Intrinsic::x86_avx512_vpdpbusds_512:
6411 case Intrinsic::x86_avx2_vpdpbssd_128:
6412 case Intrinsic::x86_avx2_vpdpbssd_256:
6413 case Intrinsic::x86_avx10_vpdpbssd_512:
6414 case Intrinsic::x86_avx2_vpdpbssds_128:
6415 case Intrinsic::x86_avx2_vpdpbssds_256:
6416 case Intrinsic::x86_avx10_vpdpbssds_512:
6417 case Intrinsic::x86_avx2_vpdpbsud_128:
6418 case Intrinsic::x86_avx2_vpdpbsud_256:
6419 case Intrinsic::x86_avx10_vpdpbsud_512:
6420 case Intrinsic::x86_avx2_vpdpbsuds_128:
6421 case Intrinsic::x86_avx2_vpdpbsuds_256:
6422 case Intrinsic::x86_avx10_vpdpbsuds_512:
6423 case Intrinsic::x86_avx2_vpdpbuud_128:
6424 case Intrinsic::x86_avx2_vpdpbuud_256:
6425 case Intrinsic::x86_avx10_vpdpbuud_512:
6426 case Intrinsic::x86_avx2_vpdpbuuds_128:
6427 case Intrinsic::x86_avx2_vpdpbuuds_256:
6428 case Intrinsic::x86_avx10_vpdpbuuds_512:
6429 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
6430 /*ZeroPurifies=*/true,
6431 /*EltSizeInBits=*/0,
6432 /*Lanes=*/kBothLanes);
6433 break;
6434
6435 // AVX Vector Neural Network Instructions: words
6436 //
6437 // Multiply and Add Signed Word Integers
6438 // < 4 x i32> @llvm.x86.avx512.vpdpwssd.128
6439 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6440 // < 8 x i32> @llvm.x86.avx512.vpdpwssd.256
6441 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6442 // <16 x i32> @llvm.x86.avx512.vpdpwssd.512
6443 // (<16 x i32>, <32 x i16>, <32 x i16>)
6444 //
6445 // Multiply and Add Signed Word Integers With Saturation
6446 // < 4 x i32> @llvm.x86.avx512.vpdpwssds.128
6447 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6448 // < 8 x i32> @llvm.x86.avx512.vpdpwssds.256
6449 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6450 // <16 x i32> @llvm.x86.avx512.vpdpwssds.512
6451 // (<16 x i32>, <32 x i16>, <32 x i16>)
6452 //
6453 // Multiply and Add Signed and Unsigned Word Integers
6454 // < 4 x i32> @llvm.x86.avx2.vpdpwsud.128
6455 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6456 // < 8 x i32> @llvm.x86.avx2.vpdpwsud.256
6457 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6458 // <16 x i32> @llvm.x86.avx10.vpdpwsud.512
6459 // (<16 x i32>, <32 x i16>, <32 x i16>)
6460 //
6461 // Multiply and Add Signed and Unsigned Word Integers With Saturation
6462 // < 4 x i32> @llvm.x86.avx2.vpdpwsuds.128
6463 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6464 // < 8 x i32> @llvm.x86.avx2.vpdpwsuds.256
6465 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6466 // <16 x i32> @llvm.x86.avx10.vpdpwsuds.512
6467 // (<16 x i32>, <32 x i16>, <32 x i16>)
6468 //
6469 // Multiply and Add Unsigned and Signed Word Integers
6470 // < 4 x i32> @llvm.x86.avx2.vpdpwusd.128
6471 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6472 // < 8 x i32> @llvm.x86.avx2.vpdpwusd.256
6473 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6474 // <16 x i32> @llvm.x86.avx10.vpdpwusd.512
6475 // (<16 x i32>, <32 x i16>, <32 x i16>)
6476 //
6477 // Multiply and Add Unsigned and Signed Word Integers With Saturation
6478 // < 4 x i32> @llvm.x86.avx2.vpdpwusds.128
6479 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6480 // < 8 x i32> @llvm.x86.avx2.vpdpwusds.256
6481 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6482 // <16 x i32> @llvm.x86.avx10.vpdpwusds.512
6483 // (<16 x i32>, <32 x i16>, <32 x i16>)
6484 //
6485 // Multiply and Add Unsigned and Unsigned Word Integers
6486 // < 4 x i32> @llvm.x86.avx2.vpdpwuud.128
6487 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6488 // < 8 x i32> @llvm.x86.avx2.vpdpwuud.256
6489 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6490 // <16 x i32> @llvm.x86.avx10.vpdpwuud.512
6491 // (<16 x i32>, <32 x i16>, <32 x i16>)
6492 //
6493 // Multiply and Add Unsigned and Unsigned Word Integers With Saturation
6494 // < 4 x i32> @llvm.x86.avx2.vpdpwuuds.128
6495 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6496 // < 8 x i32> @llvm.x86.avx2.vpdpwuuds.256
6497 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6498 // <16 x i32> @llvm.x86.avx10.vpdpwuuds.512
6499 // (<16 x i32>, <32 x i16>, <32 x i16>)
6500 //
6501 // These intrinsics are auto-upgraded into non-masked forms:
6502 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128
6503 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6504 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128
6505 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6506 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256
6507 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6508 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256
6509 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6510 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512
6511 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6512 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512
6513 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6514 //
6515 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128
6516 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6517 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128
6518 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6519 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256
6520 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6521 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256
6522 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6523 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512
6524 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6525 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512
6526 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6527 case Intrinsic::x86_avx512_vpdpwssd_128:
6528 case Intrinsic::x86_avx512_vpdpwssd_256:
6529 case Intrinsic::x86_avx512_vpdpwssd_512:
6530 case Intrinsic::x86_avx512_vpdpwssds_128:
6531 case Intrinsic::x86_avx512_vpdpwssds_256:
6532 case Intrinsic::x86_avx512_vpdpwssds_512:
6533 case Intrinsic::x86_avx2_vpdpwsud_128:
6534 case Intrinsic::x86_avx2_vpdpwsud_256:
6535 case Intrinsic::x86_avx10_vpdpwsud_512:
6536 case Intrinsic::x86_avx2_vpdpwsuds_128:
6537 case Intrinsic::x86_avx2_vpdpwsuds_256:
6538 case Intrinsic::x86_avx10_vpdpwsuds_512:
6539 case Intrinsic::x86_avx2_vpdpwusd_128:
6540 case Intrinsic::x86_avx2_vpdpwusd_256:
6541 case Intrinsic::x86_avx10_vpdpwusd_512:
6542 case Intrinsic::x86_avx2_vpdpwusds_128:
6543 case Intrinsic::x86_avx2_vpdpwusds_256:
6544 case Intrinsic::x86_avx10_vpdpwusds_512:
6545 case Intrinsic::x86_avx2_vpdpwuud_128:
6546 case Intrinsic::x86_avx2_vpdpwuud_256:
6547 case Intrinsic::x86_avx10_vpdpwuud_512:
6548 case Intrinsic::x86_avx2_vpdpwuuds_128:
6549 case Intrinsic::x86_avx2_vpdpwuuds_256:
6550 case Intrinsic::x86_avx10_vpdpwuuds_512:
6551 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6552 /*ZeroPurifies=*/true,
6553 /*EltSizeInBits=*/0,
6554 /*Lanes=*/kBothLanes);
6555 break;
6556
6557 // Dot Product of BF16 Pairs Accumulated Into Packed Single
6558 // Precision
6559 // <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128
6560 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6561 // <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256
6562 // (<8 x float>, <16 x bfloat>, <16 x bfloat>)
6563 // <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512
6564 // (<16 x float>, <32 x bfloat>, <32 x bfloat>)
6565 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6566 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6567 case Intrinsic::x86_avx512bf16_dpbf16ps_512:
6568 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6569 /*ZeroPurifies=*/false,
6570 /*EltSizeInBits=*/0,
6571 /*Lanes=*/kBothLanes);
6572 break;
6573
6574 case Intrinsic::x86_sse_cmp_ss:
6575 case Intrinsic::x86_sse2_cmp_sd:
6576 case Intrinsic::x86_sse_comieq_ss:
6577 case Intrinsic::x86_sse_comilt_ss:
6578 case Intrinsic::x86_sse_comile_ss:
6579 case Intrinsic::x86_sse_comigt_ss:
6580 case Intrinsic::x86_sse_comige_ss:
6581 case Intrinsic::x86_sse_comineq_ss:
6582 case Intrinsic::x86_sse_ucomieq_ss:
6583 case Intrinsic::x86_sse_ucomilt_ss:
6584 case Intrinsic::x86_sse_ucomile_ss:
6585 case Intrinsic::x86_sse_ucomigt_ss:
6586 case Intrinsic::x86_sse_ucomige_ss:
6587 case Intrinsic::x86_sse_ucomineq_ss:
6588 case Intrinsic::x86_sse2_comieq_sd:
6589 case Intrinsic::x86_sse2_comilt_sd:
6590 case Intrinsic::x86_sse2_comile_sd:
6591 case Intrinsic::x86_sse2_comigt_sd:
6592 case Intrinsic::x86_sse2_comige_sd:
6593 case Intrinsic::x86_sse2_comineq_sd:
6594 case Intrinsic::x86_sse2_ucomieq_sd:
6595 case Intrinsic::x86_sse2_ucomilt_sd:
6596 case Intrinsic::x86_sse2_ucomile_sd:
6597 case Intrinsic::x86_sse2_ucomigt_sd:
6598 case Intrinsic::x86_sse2_ucomige_sd:
6599 case Intrinsic::x86_sse2_ucomineq_sd:
6600 handleVectorCompareScalarIntrinsic(I);
6601 break;
6602
6603 case Intrinsic::x86_avx_cmp_pd_256:
6604 case Intrinsic::x86_avx_cmp_ps_256:
6605 case Intrinsic::x86_sse2_cmp_pd:
6606 case Intrinsic::x86_sse_cmp_ps:
6607 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/true);
6608 break;
6609
6610 case Intrinsic::x86_bmi_bextr_32:
6611 case Intrinsic::x86_bmi_bextr_64:
6612 case Intrinsic::x86_bmi_bzhi_32:
6613 case Intrinsic::x86_bmi_bzhi_64:
6614 handleGenericBitManipulation(I);
6615 break;
6616
6617 case Intrinsic::x86_pclmulqdq:
6618 case Intrinsic::x86_pclmulqdq_256:
6619 case Intrinsic::x86_pclmulqdq_512:
6620 handlePclmulIntrinsic(I);
6621 break;
6622
6623 case Intrinsic::x86_avx_round_pd_256:
6624 case Intrinsic::x86_avx_round_ps_256:
6625 case Intrinsic::x86_sse41_round_pd:
6626 case Intrinsic::x86_sse41_round_ps:
6627 handleRoundPdPsIntrinsic(I);
6628 break;
6629
6630 case Intrinsic::x86_sse41_round_sd:
6631 case Intrinsic::x86_sse41_round_ss:
6632 handleUnarySdSsIntrinsic(I);
6633 break;
6634
6635 case Intrinsic::x86_sse2_max_sd:
6636 case Intrinsic::x86_sse_max_ss:
6637 case Intrinsic::x86_sse2_min_sd:
6638 case Intrinsic::x86_sse_min_ss:
6639 handleBinarySdSsIntrinsic(I);
6640 break;
6641
6642 case Intrinsic::x86_avx_vtestc_pd:
6643 case Intrinsic::x86_avx_vtestc_pd_256:
6644 case Intrinsic::x86_avx_vtestc_ps:
6645 case Intrinsic::x86_avx_vtestc_ps_256:
6646 case Intrinsic::x86_avx_vtestnzc_pd:
6647 case Intrinsic::x86_avx_vtestnzc_pd_256:
6648 case Intrinsic::x86_avx_vtestnzc_ps:
6649 case Intrinsic::x86_avx_vtestnzc_ps_256:
6650 case Intrinsic::x86_avx_vtestz_pd:
6651 case Intrinsic::x86_avx_vtestz_pd_256:
6652 case Intrinsic::x86_avx_vtestz_ps:
6653 case Intrinsic::x86_avx_vtestz_ps_256:
6654 case Intrinsic::x86_avx_ptestc_256:
6655 case Intrinsic::x86_avx_ptestnzc_256:
6656 case Intrinsic::x86_avx_ptestz_256:
6657 case Intrinsic::x86_sse41_ptestc:
6658 case Intrinsic::x86_sse41_ptestnzc:
6659 case Intrinsic::x86_sse41_ptestz:
6660 handleVtestIntrinsic(I);
6661 break;
6662
6663 // Packed Horizontal Add/Subtract
6664 case Intrinsic::x86_ssse3_phadd_w:
6665 case Intrinsic::x86_ssse3_phadd_w_128:
6666 case Intrinsic::x86_ssse3_phsub_w:
6667 case Intrinsic::x86_ssse3_phsub_w_128:
6668 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6669 /*ReinterpretElemWidth=*/16);
6670 break;
6671
6672 case Intrinsic::x86_avx2_phadd_w:
6673 case Intrinsic::x86_avx2_phsub_w:
6674 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6675 /*ReinterpretElemWidth=*/16);
6676 break;
6677
6678 // Packed Horizontal Add/Subtract
6679 case Intrinsic::x86_ssse3_phadd_d:
6680 case Intrinsic::x86_ssse3_phadd_d_128:
6681 case Intrinsic::x86_ssse3_phsub_d:
6682 case Intrinsic::x86_ssse3_phsub_d_128:
6683 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6684 /*ReinterpretElemWidth=*/32);
6685 break;
6686
6687 case Intrinsic::x86_avx2_phadd_d:
6688 case Intrinsic::x86_avx2_phsub_d:
6689 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6690 /*ReinterpretElemWidth=*/32);
6691 break;
6692
6693 // Packed Horizontal Add/Subtract and Saturate
6694 case Intrinsic::x86_ssse3_phadd_sw:
6695 case Intrinsic::x86_ssse3_phadd_sw_128:
6696 case Intrinsic::x86_ssse3_phsub_sw:
6697 case Intrinsic::x86_ssse3_phsub_sw_128:
6698 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6699 /*ReinterpretElemWidth=*/16);
6700 break;
6701
6702 case Intrinsic::x86_avx2_phadd_sw:
6703 case Intrinsic::x86_avx2_phsub_sw:
6704 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6705 /*ReinterpretElemWidth=*/16);
6706 break;
6707
6708 // Packed Single/Double Precision Floating-Point Horizontal Add
6709 case Intrinsic::x86_sse3_hadd_ps:
6710 case Intrinsic::x86_sse3_hadd_pd:
6711 case Intrinsic::x86_sse3_hsub_ps:
6712 case Intrinsic::x86_sse3_hsub_pd:
6713 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
6714 break;
6715
6716 case Intrinsic::x86_avx_hadd_pd_256:
6717 case Intrinsic::x86_avx_hadd_ps_256:
6718 case Intrinsic::x86_avx_hsub_pd_256:
6719 case Intrinsic::x86_avx_hsub_ps_256:
6720 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2);
6721 break;
6722
6723 case Intrinsic::x86_avx_maskstore_ps:
6724 case Intrinsic::x86_avx_maskstore_pd:
6725 case Intrinsic::x86_avx_maskstore_ps_256:
6726 case Intrinsic::x86_avx_maskstore_pd_256:
6727 case Intrinsic::x86_avx2_maskstore_d:
6728 case Intrinsic::x86_avx2_maskstore_q:
6729 case Intrinsic::x86_avx2_maskstore_d_256:
6730 case Intrinsic::x86_avx2_maskstore_q_256: {
6731 handleAVXMaskedStore(I);
6732 break;
6733 }
6734
6735 case Intrinsic::x86_avx_maskload_ps:
6736 case Intrinsic::x86_avx_maskload_pd:
6737 case Intrinsic::x86_avx_maskload_ps_256:
6738 case Intrinsic::x86_avx_maskload_pd_256:
6739 case Intrinsic::x86_avx2_maskload_d:
6740 case Intrinsic::x86_avx2_maskload_q:
6741 case Intrinsic::x86_avx2_maskload_d_256:
6742 case Intrinsic::x86_avx2_maskload_q_256: {
6743 handleAVXMaskedLoad(I);
6744 break;
6745 }
6746
6747 // Packed
6748 case Intrinsic::x86_avx512fp16_add_ph_512:
6749 case Intrinsic::x86_avx512fp16_sub_ph_512:
6750 case Intrinsic::x86_avx512fp16_mul_ph_512:
6751 case Intrinsic::x86_avx512fp16_div_ph_512:
6752 case Intrinsic::x86_avx512fp16_max_ph_512:
6753 case Intrinsic::x86_avx512fp16_min_ph_512:
6754 case Intrinsic::x86_avx512_min_ps_512:
6755 case Intrinsic::x86_avx512_min_pd_512:
6756 case Intrinsic::x86_avx512_max_ps_512:
6757 case Intrinsic::x86_avx512_max_pd_512: {
6758 // These AVX512 variants contain the rounding mode as a trailing flag.
6759 // Earlier variants do not have a trailing flag and are already handled
6760 // by maybeHandleSimpleNomemIntrinsic(I, 0) via
6761 // maybeHandleUnknownIntrinsic.
6762 [[maybe_unused]] bool Success =
6763 maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/1);
6764 assert(Success);
6765 break;
6766 }
6767
6768 case Intrinsic::x86_avx_vpermilvar_pd:
6769 case Intrinsic::x86_avx_vpermilvar_pd_256:
6770 case Intrinsic::x86_avx512_vpermilvar_pd_512:
6771 case Intrinsic::x86_avx_vpermilvar_ps:
6772 case Intrinsic::x86_avx_vpermilvar_ps_256:
6773 case Intrinsic::x86_avx512_vpermilvar_ps_512: {
6774 handleAVXVpermilvar(I);
6775 break;
6776 }
6777
6778 case Intrinsic::x86_avx512_vpermi2var_d_128:
6779 case Intrinsic::x86_avx512_vpermi2var_d_256:
6780 case Intrinsic::x86_avx512_vpermi2var_d_512:
6781 case Intrinsic::x86_avx512_vpermi2var_hi_128:
6782 case Intrinsic::x86_avx512_vpermi2var_hi_256:
6783 case Intrinsic::x86_avx512_vpermi2var_hi_512:
6784 case Intrinsic::x86_avx512_vpermi2var_pd_128:
6785 case Intrinsic::x86_avx512_vpermi2var_pd_256:
6786 case Intrinsic::x86_avx512_vpermi2var_pd_512:
6787 case Intrinsic::x86_avx512_vpermi2var_ps_128:
6788 case Intrinsic::x86_avx512_vpermi2var_ps_256:
6789 case Intrinsic::x86_avx512_vpermi2var_ps_512:
6790 case Intrinsic::x86_avx512_vpermi2var_q_128:
6791 case Intrinsic::x86_avx512_vpermi2var_q_256:
6792 case Intrinsic::x86_avx512_vpermi2var_q_512:
6793 case Intrinsic::x86_avx512_vpermi2var_qi_128:
6794 case Intrinsic::x86_avx512_vpermi2var_qi_256:
6795 case Intrinsic::x86_avx512_vpermi2var_qi_512:
6796 handleAVXVpermi2var(I);
6797 break;
6798
6799 // Packed Shuffle
6800 // llvm.x86.sse.pshuf.w(<1 x i64>, i8)
6801 // llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>)
6802 // llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
6803 // llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
6804 // llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
6805 //
6806 // The following intrinsics are auto-upgraded:
6807 // llvm.x86.sse2.pshuf.d(<4 x i32>, i8)
6808 // llvm.x86.sse2.gpshufh.w(<8 x i16>, i8)
6809 // llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
6810 case Intrinsic::x86_avx2_pshuf_b:
6811 case Intrinsic::x86_sse_pshuf_w:
6812 case Intrinsic::x86_ssse3_pshuf_b_128:
6813 case Intrinsic::x86_ssse3_pshuf_b:
6814 case Intrinsic::x86_avx512_pshuf_b_512:
6815 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6816 /*trailingVerbatimArgs=*/1,
6817 /*forceIntegerIntrinsic=*/false);
6818 break;
6819
6820 // AVX512 PMOV: Packed MOV, with truncation
6821 // Precisely handled by applying the same intrinsic to the shadow
6822 case Intrinsic::x86_avx512_mask_pmov_dw_128:
6823 case Intrinsic::x86_avx512_mask_pmov_db_128:
6824 case Intrinsic::x86_avx512_mask_pmov_qb_128:
6825 case Intrinsic::x86_avx512_mask_pmov_qw_128:
6826 case Intrinsic::x86_avx512_mask_pmov_qd_128:
6827 case Intrinsic::x86_avx512_mask_pmov_wb_128:
6828 case Intrinsic::x86_avx512_mask_pmov_dw_256:
6829 case Intrinsic::x86_avx512_mask_pmov_db_256:
6830 case Intrinsic::x86_avx512_mask_pmov_qb_256:
6831 case Intrinsic::x86_avx512_mask_pmov_qw_256:
6832 case Intrinsic::x86_avx512_mask_pmov_dw_512:
6833 case Intrinsic::x86_avx512_mask_pmov_db_512:
6834 case Intrinsic::x86_avx512_mask_pmov_qb_512:
6835 case Intrinsic::x86_avx512_mask_pmov_qw_512: {
6836 // Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} were removed in
6837 // f608dc1f5775ee880e8ea30e2d06ab5a4a935c22
6838 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6839 /*trailingVerbatimArgs=*/1,
6840 /*forceIntegerIntrinsic=*/false);
6841 break;
6842 }
6843
6844 // AVX512 PMOV{S,US}: Packed MOV, with signed/unsigned saturation
6845 // Approximately handled using the corresponding truncation intrinsic
6846 // TODO: improve handleAVX512VectorDownConvert to precisely model saturation
6847 case Intrinsic::x86_avx512_mask_pmovs_dw_512:
6848 case Intrinsic::x86_avx512_mask_pmovus_dw_512: {
6849 handleIntrinsicByApplyingToShadow(
6850 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_512,
6851 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6852 break;
6853 }
6854
6855 case Intrinsic::x86_avx512_mask_pmovs_dw_256:
6856 case Intrinsic::x86_avx512_mask_pmovus_dw_256:
6857 handleIntrinsicByApplyingToShadow(
6858 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_256,
6859 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6860 break;
6861
6862 case Intrinsic::x86_avx512_mask_pmovs_dw_128:
6863 case Intrinsic::x86_avx512_mask_pmovus_dw_128:
6864 handleIntrinsicByApplyingToShadow(
6865 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_128,
6866 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6867 break;
6868
6869 case Intrinsic::x86_avx512_mask_pmovs_db_512:
6870 case Intrinsic::x86_avx512_mask_pmovus_db_512: {
6871 handleIntrinsicByApplyingToShadow(
6872 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_512,
6873 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6874 break;
6875 }
6876
6877 case Intrinsic::x86_avx512_mask_pmovs_db_256:
6878 case Intrinsic::x86_avx512_mask_pmovus_db_256:
6879 handleIntrinsicByApplyingToShadow(
6880 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_256,
6881 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6882 break;
6883
6884 case Intrinsic::x86_avx512_mask_pmovs_db_128:
6885 case Intrinsic::x86_avx512_mask_pmovus_db_128:
6886 handleIntrinsicByApplyingToShadow(
6887 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_128,
6888 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6889 break;
6890
6891 case Intrinsic::x86_avx512_mask_pmovs_qb_512:
6892 case Intrinsic::x86_avx512_mask_pmovus_qb_512: {
6893 handleIntrinsicByApplyingToShadow(
6894 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_512,
6895 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6896 break;
6897 }
6898
6899 case Intrinsic::x86_avx512_mask_pmovs_qb_256:
6900 case Intrinsic::x86_avx512_mask_pmovus_qb_256:
6901 handleIntrinsicByApplyingToShadow(
6902 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_256,
6903 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6904 break;
6905
6906 case Intrinsic::x86_avx512_mask_pmovs_qb_128:
6907 case Intrinsic::x86_avx512_mask_pmovus_qb_128:
6908 handleIntrinsicByApplyingToShadow(
6909 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_128,
6910 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6911 break;
6912
6913 case Intrinsic::x86_avx512_mask_pmovs_qw_512:
6914 case Intrinsic::x86_avx512_mask_pmovus_qw_512: {
6915 handleIntrinsicByApplyingToShadow(
6916 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_512,
6917 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6918 break;
6919 }
6920
6921 case Intrinsic::x86_avx512_mask_pmovs_qw_256:
6922 case Intrinsic::x86_avx512_mask_pmovus_qw_256:
6923 handleIntrinsicByApplyingToShadow(
6924 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_256,
6925 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6926 break;
6927
6928 case Intrinsic::x86_avx512_mask_pmovs_qw_128:
6929 case Intrinsic::x86_avx512_mask_pmovus_qw_128:
6930 handleIntrinsicByApplyingToShadow(
6931 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_128,
6932 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6933 break;
6934
6935 case Intrinsic::x86_avx512_mask_pmovs_qd_128:
6936 case Intrinsic::x86_avx512_mask_pmovus_qd_128:
6937 handleIntrinsicByApplyingToShadow(
6938 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qd_128,
6939 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6940 break;
6941
6942 case Intrinsic::x86_avx512_mask_pmovs_wb_128:
6943 case Intrinsic::x86_avx512_mask_pmovus_wb_128:
6944 handleIntrinsicByApplyingToShadow(
6945 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_wb_128,
6946 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6947 break;
6948
6949 case Intrinsic::x86_avx512_mask_pmovs_qd_256:
6950 case Intrinsic::x86_avx512_mask_pmovus_qd_256:
6951 case Intrinsic::x86_avx512_mask_pmovs_wb_256:
6952 case Intrinsic::x86_avx512_mask_pmovus_wb_256:
6953 case Intrinsic::x86_avx512_mask_pmovs_qd_512:
6954 case Intrinsic::x86_avx512_mask_pmovus_qd_512:
6955 case Intrinsic::x86_avx512_mask_pmovs_wb_512:
6956 case Intrinsic::x86_avx512_mask_pmovus_wb_512: {
6957 // Since Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} do not exist,
6958 // we cannot use handleIntrinsicByApplyingToShadow. Instead, we call the
6959 // slow-path handler.
6960 handleAVX512VectorDownConvert(I);
6961 break;
6962 }
6963
6964 // e.g.,
6965 // <16 x float> @llvm.x86.avx512.mask.compress
6966 // (<16 x float> %data, <16 x float> %passthru,
6967 // <16 x i1> %mask)
6968 // <16 x i32> @llvm.x86.avx512.mask.compress
6969 // (<16 x i32> %data, <16 x i32> %passthru,
6970 // <16 x i1> %mask)
6971 case Intrinsic::x86_avx512_mask_compress:
6972 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6973 /*trailingVerbatimArgs=*/1,
6974 /*forceIntegerIntrinsic=*/true);
6975 break;
6976
6977 // AVX512/AVX10 Reciprocal
6978 // <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
6979 // (<16 x float>, <16 x float>, i16)
6980 // <8 x float> @llvm.x86.avx512.rsqrt14.ps.256
6981 // (<8 x float>, <8 x float>, i8)
6982 // <4 x float> @llvm.x86.avx512.rsqrt14.ps.128
6983 // (<4 x float>, <4 x float>, i8)
6984 //
6985 // <8 x double> @llvm.x86.avx512.rsqrt14.pd.512
6986 // (<8 x double>, <8 x double>, i8)
6987 // <4 x double> @llvm.x86.avx512.rsqrt14.pd.256
6988 // (<4 x double>, <4 x double>, i8)
6989 // <2 x double> @llvm.x86.avx512.rsqrt14.pd.128
6990 // (<2 x double>, <2 x double>, i8)
6991 //
6992 // <32 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.512
6993 // (<32 x bfloat>, <32 x bfloat>, i32)
6994 // <16 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.256
6995 // (<16 x bfloat>, <16 x bfloat>, i16)
6996 // <8 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.128
6997 // (<8 x bfloat>, <8 x bfloat>, i8)
6998 //
6999 // <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512
7000 // (<32 x half>, <32 x half>, i32)
7001 // <16 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.256
7002 // (<16 x half>, <16 x half>, i16)
7003 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.128
7004 // (<8 x half>, <8 x half>, i8)
7005 //
7006 // TODO: 3-operand variants are not handled:
7007 // <2 x double> @llvm.x86.avx512.rsqrt14.sd
7008 // (<2 x double>, <2 x double>, <2 x double>, i8)
7009 // <4 x float> @llvm.x86.avx512.rsqrt14.ss
7010 // (<4 x float>, <4 x float>, <4 x float>, i8)
7011 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh
7012 // (<8 x half>, <8 x half>, <8 x half>, i8)
7013 case Intrinsic::x86_avx512_rsqrt14_ps_512:
7014 case Intrinsic::x86_avx512_rsqrt14_ps_256:
7015 case Intrinsic::x86_avx512_rsqrt14_ps_128:
7016 case Intrinsic::x86_avx512_rsqrt14_pd_512:
7017 case Intrinsic::x86_avx512_rsqrt14_pd_256:
7018 case Intrinsic::x86_avx512_rsqrt14_pd_128:
7019 case Intrinsic::x86_avx10_mask_rsqrt_bf16_512:
7020 case Intrinsic::x86_avx10_mask_rsqrt_bf16_256:
7021 case Intrinsic::x86_avx10_mask_rsqrt_bf16_128:
7022 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_512:
7023 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_256:
7024 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_128:
7025 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7026 /*WriteThruIndex=*/1,
7027 /*MaskIndex=*/2);
7028 break;
7029
7030 // AVX512/AVX10 Reciprocal Square Root
7031 // <16 x float> @llvm.x86.avx512.rcp14.ps.512
7032 // (<16 x float>, <16 x float>, i16)
7033 // <8 x float> @llvm.x86.avx512.rcp14.ps.256
7034 // (<8 x float>, <8 x float>, i8)
7035 // <4 x float> @llvm.x86.avx512.rcp14.ps.128
7036 // (<4 x float>, <4 x float>, i8)
7037 //
7038 // <8 x double> @llvm.x86.avx512.rcp14.pd.512
7039 // (<8 x double>, <8 x double>, i8)
7040 // <4 x double> @llvm.x86.avx512.rcp14.pd.256
7041 // (<4 x double>, <4 x double>, i8)
7042 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
7043 // (<2 x double>, <2 x double>, i8)
7044 //
7045 // <32 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.512
7046 // (<32 x bfloat>, <32 x bfloat>, i32)
7047 // <16 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.256
7048 // (<16 x bfloat>, <16 x bfloat>, i16)
7049 // <8 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.128
7050 // (<8 x bfloat>, <8 x bfloat>, i8)
7051 //
7052 // <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512
7053 // (<32 x half>, <32 x half>, i32)
7054 // <16 x half> @llvm.x86.avx512fp16.mask.rcp.ph.256
7055 // (<16 x half>, <16 x half>, i16)
7056 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.ph.128
7057 // (<8 x half>, <8 x half>, i8)
7058 //
7059 // TODO: 3-operand variants are not handled:
7060 // <2 x double> @llvm.x86.avx512.rcp14.sd
7061 // (<2 x double>, <2 x double>, <2 x double>, i8)
7062 // <4 x float> @llvm.x86.avx512.rcp14.ss
7063 // (<4 x float>, <4 x float>, <4 x float>, i8)
7064 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh
7065 // (<8 x half>, <8 x half>, <8 x half>, i8)
7066 case Intrinsic::x86_avx512_rcp14_ps_512:
7067 case Intrinsic::x86_avx512_rcp14_ps_256:
7068 case Intrinsic::x86_avx512_rcp14_ps_128:
7069 case Intrinsic::x86_avx512_rcp14_pd_512:
7070 case Intrinsic::x86_avx512_rcp14_pd_256:
7071 case Intrinsic::x86_avx512_rcp14_pd_128:
7072 case Intrinsic::x86_avx10_mask_rcp_bf16_512:
7073 case Intrinsic::x86_avx10_mask_rcp_bf16_256:
7074 case Intrinsic::x86_avx10_mask_rcp_bf16_128:
7075 case Intrinsic::x86_avx512fp16_mask_rcp_ph_512:
7076 case Intrinsic::x86_avx512fp16_mask_rcp_ph_256:
7077 case Intrinsic::x86_avx512fp16_mask_rcp_ph_128:
7078 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7079 /*WriteThruIndex=*/1,
7080 /*MaskIndex=*/2);
7081 break;
7082
7083 // <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512
7084 // (<32 x half>, i32, <32 x half>, i32, i32)
7085 // <16 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.256
7086 // (<16 x half>, i32, <16 x half>, i32, i16)
7087 // <8 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.128
7088 // (<8 x half>, i32, <8 x half>, i32, i8)
7089 //
7090 // <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512
7091 // (<16 x float>, i32, <16 x float>, i16, i32)
7092 // <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256
7093 // (<8 x float>, i32, <8 x float>, i8)
7094 // <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128
7095 // (<4 x float>, i32, <4 x float>, i8)
7096 //
7097 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
7098 // (<8 x double>, i32, <8 x double>, i8, i32)
7099 // A Imm WriteThru Mask Rounding
7100 // <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256
7101 // (<4 x double>, i32, <4 x double>, i8)
7102 // <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128
7103 // (<2 x double>, i32, <2 x double>, i8)
7104 // A Imm WriteThru Mask
7105 //
7106 // <32 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.512
7107 // (<32 x bfloat>, i32, <32 x bfloat>, i32)
7108 // <16 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.256
7109 // (<16 x bfloat>, i32, <16 x bfloat>, i16)
7110 // <8 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.128
7111 // (<8 x bfloat>, i32, <8 x bfloat>, i8)
7112 //
7113 // Not supported: three vectors
7114 // - <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh
7115 // (<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
7116 // - <4 x float> @llvm.x86.avx512.mask.rndscale.ss
7117 // (<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
7118 // - <2 x double> @llvm.x86.avx512.mask.rndscale.sd
7119 // (<2 x double>, <2 x double>, <2 x double>, i8, i32,
7120 // i32)
7121 // A B WriteThru Mask Imm
7122 // Rounding
7123 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_512:
7124 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_256:
7125 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_128:
7126 case Intrinsic::x86_avx512_mask_rndscale_ps_512:
7127 case Intrinsic::x86_avx512_mask_rndscale_ps_256:
7128 case Intrinsic::x86_avx512_mask_rndscale_ps_128:
7129 case Intrinsic::x86_avx512_mask_rndscale_pd_512:
7130 case Intrinsic::x86_avx512_mask_rndscale_pd_256:
7131 case Intrinsic::x86_avx512_mask_rndscale_pd_128:
7132 case Intrinsic::x86_avx10_mask_rndscale_bf16_512:
7133 case Intrinsic::x86_avx10_mask_rndscale_bf16_256:
7134 case Intrinsic::x86_avx10_mask_rndscale_bf16_128:
7135 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7136 /*WriteThruIndex=*/2,
7137 /*MaskIndex=*/3);
7138 break;
7139
7140 // AVX512 Vector Scale Float* Packed
7141 //
7142 // < 8 x double> @llvm.x86.avx512.mask.scalef.pd.512
7143 // (<8 x double>, <8 x double>, <8 x double>, i8, i32)
7144 // A B WriteThru Msk Round
7145 // < 4 x double> @llvm.x86.avx512.mask.scalef.pd.256
7146 // (<4 x double>, <4 x double>, <4 x double>, i8)
7147 // < 2 x double> @llvm.x86.avx512.mask.scalef.pd.128
7148 // (<2 x double>, <2 x double>, <2 x double>, i8)
7149 //
7150 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
7151 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
7152 // < 8 x float> @llvm.x86.avx512.mask.scalef.ps.256
7153 // (<8 x float>, <8 x float>, <8 x float>, i8)
7154 // < 4 x float> @llvm.x86.avx512.mask.scalef.ps.128
7155 // (<4 x float>, <4 x float>, <4 x float>, i8)
7156 //
7157 // <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512
7158 // (<32 x half>, <32 x half>, <32 x half>, i32, i32)
7159 // <16 x half> @llvm.x86.avx512fp16.mask.scalef.ph.256
7160 // (<16 x half>, <16 x half>, <16 x half>, i16)
7161 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.ph.128
7162 // (<8 x half>, <8 x half>, <8 x half>, i8)
7163 //
7164 // TODO: AVX10
7165 // <32 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.512
7166 // (<32 x bfloat>, <32 x bfloat>, <32 x bfloat>, i32)
7167 // <16 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.256
7168 // (<16 x bfloat>, <16 x bfloat>, <16 x bfloat>, i16)
7169 // < 8 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.128
7170 // (<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i8)
7171 case Intrinsic::x86_avx512_mask_scalef_pd_512:
7172 case Intrinsic::x86_avx512_mask_scalef_pd_256:
7173 case Intrinsic::x86_avx512_mask_scalef_pd_128:
7174 case Intrinsic::x86_avx512_mask_scalef_ps_512:
7175 case Intrinsic::x86_avx512_mask_scalef_ps_256:
7176 case Intrinsic::x86_avx512_mask_scalef_ps_128:
7177 case Intrinsic::x86_avx512fp16_mask_scalef_ph_512:
7178 case Intrinsic::x86_avx512fp16_mask_scalef_ph_256:
7179 case Intrinsic::x86_avx512fp16_mask_scalef_ph_128:
7180 // The AVX512 512-bit operand variants have an extra operand (the
7181 // Rounding mode). The extra operand, if present, will be
7182 // automatically checked by the handler.
7183 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0, 1},
7184 /*WriteThruIndex=*/2,
7185 /*MaskIndex=*/3);
7186 break;
7187
7188 // TODO: AVX512 Vector Scale Float* Scalar
7189 //
7190 // This is different from the Packed variant, because some bits are copied,
7191 // and some bits are zeroed.
7192 //
7193 // < 4 x float> @llvm.x86.avx512.mask.scalef.ss
7194 // (<4 x float>, <4 x float>, <4 x float>, i8, i32)
7195 //
7196 // < 2 x double> @llvm.x86.avx512.mask.scalef.sd
7197 // (<2 x double>, <2 x double>, <2 x double>, i8, i32)
7198 //
7199 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.sh
7200 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
7201
7202 // AVX512 FP16 Arithmetic
7203 case Intrinsic::x86_avx512fp16_mask_add_sh_round:
7204 case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
7205 case Intrinsic::x86_avx512fp16_mask_mul_sh_round:
7206 case Intrinsic::x86_avx512fp16_mask_div_sh_round:
7207 case Intrinsic::x86_avx512fp16_mask_max_sh_round:
7208 case Intrinsic::x86_avx512fp16_mask_min_sh_round: {
7209 visitGenericScalarHalfwordInst(I);
7210 break;
7211 }
7212
7213 // AVX512 Floating-Point Classification
7214 // - <8 x i1> @llvm.x86.avx512.fpclass.pd.512(<8 x double>, i32)
7215 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512(<16 x float>, i32)
7216 case Intrinsic::x86_avx512_fpclass_pd_512:
7217 case Intrinsic::x86_avx512_fpclass_ps_512:
7218 handleAVX512FPClass(I);
7219 break;
7220
7221 // AVX Galois Field New Instructions
7222 case Intrinsic::x86_vgf2p8affineqb_128:
7223 case Intrinsic::x86_vgf2p8affineqb_256:
7224 case Intrinsic::x86_vgf2p8affineqb_512:
7225 handleAVXGF2P8Affine(I);
7226 break;
7227
7228 default:
7229 return false;
7230 }
7231
7232 return true;
7233 }
7234
7235 bool maybeHandleArmSIMDIntrinsic(IntrinsicInst &I) {
7236 switch (I.getIntrinsicID()) {
7237 // Two operands e.g.,
7238 // - <8 x i8> @llvm.aarch64.neon.rshrn.v8i8 (<8 x i16>, i32)
7239 // - <4 x i16> @llvm.aarch64.neon.uqrshl.v4i16(<4 x i16>, <4 x i16>)
7240 case Intrinsic::aarch64_neon_rshrn:
7241 case Intrinsic::aarch64_neon_sqrshl:
7242 case Intrinsic::aarch64_neon_sqrshrn:
7243 case Intrinsic::aarch64_neon_sqrshrun:
7244 case Intrinsic::aarch64_neon_sqshl:
7245 case Intrinsic::aarch64_neon_sqshlu:
7246 case Intrinsic::aarch64_neon_sqshrn:
7247 case Intrinsic::aarch64_neon_sqshrun:
7248 case Intrinsic::aarch64_neon_srshl:
7249 case Intrinsic::aarch64_neon_sshl:
7250 case Intrinsic::aarch64_neon_uqrshl:
7251 case Intrinsic::aarch64_neon_uqrshrn:
7252 case Intrinsic::aarch64_neon_uqshl:
7253 case Intrinsic::aarch64_neon_uqshrn:
7254 case Intrinsic::aarch64_neon_urshl:
7255 case Intrinsic::aarch64_neon_ushl:
7256 handleVectorShiftIntrinsic(I, /* Variable */ false);
7257 break;
7258
7259 // Vector Shift Left/Right and Insert
7260 //
7261 // Three operands e.g.,
7262 // - <4 x i16> @llvm.aarch64.neon.vsli.v4i16
7263 // (<4 x i16> %a, <4 x i16> %b, i32 %n)
7264 // - <16 x i8> @llvm.aarch64.neon.vsri.v16i8
7265 // (<16 x i8> %a, <16 x i8> %b, i32 %n)
7266 //
7267 // %b is shifted by %n bits, and the "missing" bits are filled in with %a
7268 // (instead of zero-extending/sign-extending).
7269 case Intrinsic::aarch64_neon_vsli:
7270 case Intrinsic::aarch64_neon_vsri:
7271 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7272 /*trailingVerbatimArgs=*/1,
7273 /*forceIntegerIntrinsic=*/false);
7274 break;
7275
7276 // TODO: handling max/min similarly to AND/OR may be more precise
7277 // Floating-Point Maximum/Minimum Pairwise
7278 case Intrinsic::aarch64_neon_fmaxp:
7279 case Intrinsic::aarch64_neon_fminp:
7280 // Floating-Point Maximum/Minimum Number Pairwise
7281 case Intrinsic::aarch64_neon_fmaxnmp:
7282 case Intrinsic::aarch64_neon_fminnmp:
7283 // Signed/Unsigned Maximum/Minimum Pairwise
7284 case Intrinsic::aarch64_neon_smaxp:
7285 case Intrinsic::aarch64_neon_sminp:
7286 case Intrinsic::aarch64_neon_umaxp:
7287 case Intrinsic::aarch64_neon_uminp:
7288 // Add Pairwise
7289 case Intrinsic::aarch64_neon_addp:
7290 // Floating-point Add Pairwise
7291 case Intrinsic::aarch64_neon_faddp:
7292 // Add Long Pairwise
7293 case Intrinsic::aarch64_neon_saddlp:
7294 case Intrinsic::aarch64_neon_uaddlp: {
7295 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
7296 break;
7297 }
7298
7299 // Floating-point Convert to integer, rounding to nearest with ties to Away
7300 case Intrinsic::aarch64_neon_fcvtas:
7301 case Intrinsic::aarch64_neon_fcvtau:
7302 // Floating-point convert to integer, rounding toward minus infinity
7303 case Intrinsic::aarch64_neon_fcvtms:
7304 case Intrinsic::aarch64_neon_fcvtmu:
7305 // Floating-point convert to integer, rounding to nearest with ties to even
7306 case Intrinsic::aarch64_neon_fcvtns:
7307 case Intrinsic::aarch64_neon_fcvtnu:
7308 // Floating-point convert to integer, rounding toward plus infinity
7309 case Intrinsic::aarch64_neon_fcvtps:
7310 case Intrinsic::aarch64_neon_fcvtpu:
7311 // Floating-point Convert to integer, rounding toward Zero
7312 case Intrinsic::aarch64_neon_fcvtzs:
7313 case Intrinsic::aarch64_neon_fcvtzu:
7314 // Floating-point convert to lower precision narrow, rounding to odd
7315 case Intrinsic::aarch64_neon_fcvtxn:
7316 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
7317 break;
7318
7319 // Vector Conversions Between Fixed-Point and Floating-Point
7320 case Intrinsic::aarch64_neon_vcvtfxs2fp:
7321 case Intrinsic::aarch64_neon_vcvtfp2fxs:
7322 case Intrinsic::aarch64_neon_vcvtfxu2fp:
7323 case Intrinsic::aarch64_neon_vcvtfp2fxu:
7324 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/true);
7325 break;
7326
7327 // TODO: bfloat conversions
7328 // - bfloat @llvm.aarch64.neon.bfcvt(float)
7329 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float>)
7330 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat>, <4 x float>)
7331
7332 // Add reduction to scalar
7333 case Intrinsic::aarch64_neon_faddv:
7334 case Intrinsic::aarch64_neon_saddv:
7335 case Intrinsic::aarch64_neon_uaddv:
7336 // Signed/Unsigned min/max (Vector)
7337 // TODO: handling similarly to AND/OR may be more precise.
7338 case Intrinsic::aarch64_neon_smaxv:
7339 case Intrinsic::aarch64_neon_sminv:
7340 case Intrinsic::aarch64_neon_umaxv:
7341 case Intrinsic::aarch64_neon_uminv:
7342 // Floating-point min/max (vector)
7343 // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
7344 // but our shadow propagation is the same.
7345 case Intrinsic::aarch64_neon_fmaxv:
7346 case Intrinsic::aarch64_neon_fminv:
7347 case Intrinsic::aarch64_neon_fmaxnmv:
7348 case Intrinsic::aarch64_neon_fminnmv:
7349 // Sum long across vector
7350 case Intrinsic::aarch64_neon_saddlv:
7351 case Intrinsic::aarch64_neon_uaddlv:
7352 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/true);
7353 break;
7354
7355 case Intrinsic::aarch64_neon_ld1x2:
7356 case Intrinsic::aarch64_neon_ld1x3:
7357 case Intrinsic::aarch64_neon_ld1x4:
7358 case Intrinsic::aarch64_neon_ld2:
7359 case Intrinsic::aarch64_neon_ld3:
7360 case Intrinsic::aarch64_neon_ld4:
7361 case Intrinsic::aarch64_neon_ld2r:
7362 case Intrinsic::aarch64_neon_ld3r:
7363 case Intrinsic::aarch64_neon_ld4r: {
7364 handleNEONVectorLoad(I, /*WithLane=*/false);
7365 break;
7366 }
7367
7368 case Intrinsic::aarch64_neon_ld2lane:
7369 case Intrinsic::aarch64_neon_ld3lane:
7370 case Intrinsic::aarch64_neon_ld4lane: {
7371 handleNEONVectorLoad(I, /*WithLane=*/true);
7372 break;
7373 }
7374
7375 // Saturating extract narrow
7376 case Intrinsic::aarch64_neon_sqxtn:
7377 case Intrinsic::aarch64_neon_sqxtun:
7378 case Intrinsic::aarch64_neon_uqxtn:
7379 // These only have one argument, but we (ab)use handleShadowOr because it
7380 // does work on single argument intrinsics and will typecast the shadow
7381 // (and update the origin).
7382 handleShadowOr(I);
7383 break;
7384
7385 case Intrinsic::aarch64_neon_st1x2:
7386 case Intrinsic::aarch64_neon_st1x3:
7387 case Intrinsic::aarch64_neon_st1x4:
7388 case Intrinsic::aarch64_neon_st2:
7389 case Intrinsic::aarch64_neon_st3:
7390 case Intrinsic::aarch64_neon_st4: {
7391 handleNEONVectorStoreIntrinsic(I, useLane: false);
7392 break;
7393 }
7394
7395 case Intrinsic::aarch64_neon_st2lane:
7396 case Intrinsic::aarch64_neon_st3lane:
7397 case Intrinsic::aarch64_neon_st4lane: {
7398 handleNEONVectorStoreIntrinsic(I, useLane: true);
7399 break;
7400 }
7401
7402 // Arm NEON vector table intrinsics have the source/table register(s) as
7403 // arguments, followed by the index register. They return the output.
7404 //
7405 // 'TBL writes a zero if an index is out-of-range, while TBX leaves the
7406 // original value unchanged in the destination register.'
7407 // Conveniently, zero denotes a clean shadow, which means out-of-range
7408 // indices for TBL will initialize the user data with zero and also clean
7409 // the shadow. (For TBX, neither the user data nor the shadow will be
7410 // updated, which is also correct.)
7411 case Intrinsic::aarch64_neon_tbl1:
7412 case Intrinsic::aarch64_neon_tbl2:
7413 case Intrinsic::aarch64_neon_tbl3:
7414 case Intrinsic::aarch64_neon_tbl4:
7415 case Intrinsic::aarch64_neon_tbx1:
7416 case Intrinsic::aarch64_neon_tbx2:
7417 case Intrinsic::aarch64_neon_tbx3:
7418 case Intrinsic::aarch64_neon_tbx4: {
7419 // The last trailing argument (index register) should be handled verbatim
7420 handleIntrinsicByApplyingToShadow(
7421 I, /*shadowIntrinsicID=*/I.getIntrinsicID(),
7422 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
7423 break;
7424 }
7425
7426 case Intrinsic::aarch64_neon_fmulx:
7427 case Intrinsic::aarch64_neon_pmul:
7428 case Intrinsic::aarch64_neon_pmull:
7429 case Intrinsic::aarch64_neon_smull:
7430 case Intrinsic::aarch64_neon_pmull64:
7431 case Intrinsic::aarch64_neon_umull: {
7432 handleNEONVectorMultiplyIntrinsic(I);
7433 break;
7434 }
7435
7436 case Intrinsic::aarch64_neon_smmla:
7437 case Intrinsic::aarch64_neon_ummla:
7438 case Intrinsic::aarch64_neon_usmmla:
7439 case Intrinsic::aarch64_neon_bfmmla:
7440 handleNEONMatrixMultiply(I);
7441 break;
7442
7443 // <2 x i32> @llvm.aarch64.neon.{u,s,us}dot.v2i32.v8i8
7444 // (<2 x i32> %acc, <8 x i8> %a, <8 x i8> %b)
7445 // <4 x i32> @llvm.aarch64.neon.{u,s,us}dot.v4i32.v16i8
7446 // (<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b)
7447 case Intrinsic::aarch64_neon_sdot:
7448 case Intrinsic::aarch64_neon_udot:
7449 case Intrinsic::aarch64_neon_usdot:
7450 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7451 /*ZeroPurifies=*/true,
7452 /*EltSizeInBits=*/0,
7453 /*Lanes=*/kBothLanes);
7454 break;
7455
7456 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
7457 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
7458 // <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16
7459 // (<4 x float> %acc, <8 x bfloat> %a, <8 x bfloat> %b)
7460 case Intrinsic::aarch64_neon_bfdot:
7461 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7462 /*ZeroPurifies=*/false,
7463 /*EltSizeInBits=*/0,
7464 /*Lanes=*/kBothLanes);
7465 break;
7466
7467 // Floating-Point Absolute Compare Greater Than/Equal
7468 case Intrinsic::aarch64_neon_facge:
7469 case Intrinsic::aarch64_neon_facgt:
7470 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/false);
7471 break;
7472
7473 default:
7474 return false;
7475 }
7476
7477 return true;
7478 }
7479
7480 void visitIntrinsicInst(IntrinsicInst &I) {
7481 if (maybeHandleCrossPlatformIntrinsic(I))
7482 return;
7483
7484 if (maybeHandleX86SIMDIntrinsic(I))
7485 return;
7486
7487 if (maybeHandleArmSIMDIntrinsic(I))
7488 return;
7489
7490 if (maybeHandleUnknownIntrinsic(I))
7491 return;
7492
7493 visitInstruction(I);
7494 }
7495
7496 void visitLibAtomicLoad(CallBase &CB) {
7497 // Since we use getNextNode here, we can't have CB terminate the BB.
7498 assert(isa<CallInst>(CB));
7499
7500 IRBuilder<> IRB(&CB);
7501 Value *Size = CB.getArgOperand(i: 0);
7502 Value *SrcPtr = CB.getArgOperand(i: 1);
7503 Value *DstPtr = CB.getArgOperand(i: 2);
7504 Value *Ordering = CB.getArgOperand(i: 3);
7505 // Convert the call to have at least Acquire ordering to make sure
7506 // the shadow operations aren't reordered before it.
7507 Value *NewOrdering =
7508 IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
7509 CB.setArgOperand(i: 3, v: NewOrdering);
7510
7511 NextNodeIRBuilder NextIRB(&CB);
7512 Value *SrcShadowPtr, *SrcOriginPtr;
7513 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
7514 getShadowOriginPtr(Addr: SrcPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7515 /*isStore*/ false);
7516 Value *DstShadowPtr =
7517 getShadowOriginPtr(Addr: DstPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7518 /*isStore*/ true)
7519 .first;
7520
7521 NextIRB.CreateMemCpy(Dst: DstShadowPtr, DstAlign: Align(1), Src: SrcShadowPtr, SrcAlign: Align(1), Size);
7522 if (MS.TrackOrigins) {
7523 Value *SrcOrigin = NextIRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr,
7524 Align: kMinOriginAlignment);
7525 Value *NewOrigin = updateOrigin(V: SrcOrigin, IRB&: NextIRB);
7526 NextIRB.CreateCall(Callee: MS.MsanSetOriginFn, Args: {DstPtr, Size, NewOrigin});
7527 }
7528 }
7529
7530 void visitLibAtomicStore(CallBase &CB) {
7531 IRBuilder<> IRB(&CB);
7532 Value *Size = CB.getArgOperand(i: 0);
7533 Value *DstPtr = CB.getArgOperand(i: 2);
7534 Value *Ordering = CB.getArgOperand(i: 3);
7535 // Convert the call to have at least Release ordering to make sure
7536 // the shadow operations aren't reordered after it.
7537 Value *NewOrdering =
7538 IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
7539 CB.setArgOperand(i: 3, v: NewOrdering);
7540
7541 Value *DstShadowPtr =
7542 getShadowOriginPtr(Addr: DstPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1),
7543 /*isStore*/ true)
7544 .first;
7545
7546 // Atomic store always paints clean shadow/origin. See file header.
7547 IRB.CreateMemSet(Ptr: DstShadowPtr, Val: getCleanShadow(OrigTy: IRB.getInt8Ty()), Size,
7548 Align: Align(1));
7549 }
7550
7551 void visitCallBase(CallBase &CB) {
7552 assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
7553 if (CB.isInlineAsm()) {
7554 // For inline asm (either a call to asm function, or callbr instruction),
7555 // do the usual thing: check argument shadow and mark all outputs as
7556 // clean. Note that any side effects of the inline asm that are not
7557 // immediately visible in its constraints are not handled.
7558 if (ClHandleAsmConservative)
7559 visitAsmInstruction(I&: CB);
7560 else
7561 visitInstruction(I&: CB);
7562 return;
7563 }
7564 LibFunc LF = TLI->getLibFunc(CB);
7565 if (LF != NotLibFunc) {
7566 // libatomic.a functions need to have special handling because there isn't
7567 // a good way to intercept them or compile the library with
7568 // instrumentation.
7569 switch (LF) {
7570 case LibFunc_atomic_load:
7571 if (!isa<CallInst>(Val: CB)) {
7572 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
7573 "Ignoring!\n";
7574 break;
7575 }
7576 visitLibAtomicLoad(CB);
7577 return;
7578 case LibFunc_atomic_store:
7579 visitLibAtomicStore(CB);
7580 return;
7581 default:
7582 break;
7583 }
7584 }
7585
7586 if (auto *Call = dyn_cast<CallInst>(Val: &CB)) {
7587 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
7588
7589 // We are going to insert code that relies on the fact that the callee
7590 // will become a non-readonly function after it is instrumented by us. To
7591 // prevent this code from being optimized out, mark that function
7592 // non-readonly in advance.
7593 // TODO: We can likely do better than dropping memory() completely here.
7594 AttributeMask B;
7595 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
7596
7597 Call->removeFnAttrs(AttrsToRemove: B);
7598 if (Function *Func = Call->getCalledFunction()) {
7599 Func->removeFnAttrs(Attrs: B);
7600 }
7601
7602 maybeMarkSanitizerLibraryCallNoBuiltin(CI: Call, TLI);
7603 }
7604 IRBuilder<> IRB(&CB);
7605 bool MayCheckCall = MS.EagerChecks;
7606 if (Function *Func = CB.getCalledFunction()) {
7607 // __sanitizer_unaligned_{load,store} functions may be called by users
7608 // and always expects shadows in the TLS. So don't check them.
7609 MayCheckCall &= !Func->getName().starts_with(Prefix: "__sanitizer_unaligned_");
7610 }
7611
7612 unsigned ArgOffset = 0;
7613 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
7614 for (const auto &[i, A] : llvm::enumerate(First: CB.args())) {
7615 if (!A->getType()->isSized()) {
7616 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
7617 continue;
7618 }
7619
7620 if (A->getType()->isScalableTy()) {
7621 LLVM_DEBUG(dbgs() << "Arg " << i << " is vscale: " << CB << "\n");
7622 // Handle as noundef, but don't reserve tls slots.
7623 insertCheckShadowOf(Val: A, OrigIns: &CB);
7624 continue;
7625 }
7626
7627 unsigned Size = 0;
7628 const DataLayout &DL = F.getDataLayout();
7629
7630 bool ByVal = CB.isByValArgument(ArgNo: i);
7631 bool NoUndef = CB.paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef);
7632 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
7633
7634 if (EagerCheck) {
7635 insertCheckShadowOf(Val: A, OrigIns: &CB);
7636 Size = DL.getTypeAllocSize(Ty: A->getType());
7637 } else {
7638 [[maybe_unused]] Value *Store = nullptr;
7639 // Compute the Shadow for arg even if it is ByVal, because
7640 // in that case getShadow() will copy the actual arg shadow to
7641 // __msan_param_tls.
7642 Value *ArgShadow = getShadow(V: A);
7643 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
7644 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
7645 << " Shadow: " << *ArgShadow << "\n");
7646 if (ByVal) {
7647 // ByVal requires some special handling as it's too big for a single
7648 // load
7649 assert(A->getType()->isPointerTy() &&
7650 "ByVal argument is not a pointer!");
7651 Size = DL.getTypeAllocSize(Ty: CB.getParamByValType(ArgNo: i));
7652 if (ArgOffset + Size > kParamTLSSize)
7653 break;
7654 const MaybeAlign ParamAlignment(CB.getParamAlign(ArgNo: i));
7655 MaybeAlign Alignment = std::nullopt;
7656 if (ParamAlignment)
7657 Alignment = std::min(a: *ParamAlignment, b: kShadowTLSAlignment);
7658 Value *AShadowPtr, *AOriginPtr;
7659 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
7660 getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
7661 /*isStore*/ false);
7662 if (!PropagateShadow) {
7663 Store = IRB.CreateMemSet(Ptr: ArgShadowBase,
7664 Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
7665 Size, Align: Alignment);
7666 } else {
7667 Store = IRB.CreateMemCpy(Dst: ArgShadowBase, DstAlign: Alignment, Src: AShadowPtr,
7668 SrcAlign: Alignment, Size);
7669 if (MS.TrackOrigins) {
7670 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
7671 // FIXME: OriginSize should be:
7672 // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
7673 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
7674 IRB.CreateMemCpy(
7675 Dst: ArgOriginBase,
7676 /* by origin_tls[ArgOffset] */ DstAlign: kMinOriginAlignment,
7677 Src: AOriginPtr,
7678 /* by getShadowOriginPtr */ SrcAlign: kMinOriginAlignment, Size: OriginSize);
7679 }
7680 }
7681 } else {
7682 // Any other parameters mean we need bit-grained tracking of uninit
7683 // data
7684 Size = DL.getTypeAllocSize(Ty: A->getType());
7685 if (ArgOffset + Size > kParamTLSSize)
7686 break;
7687 Store = IRB.CreateAlignedStore(Val: ArgShadow, Ptr: ArgShadowBase,
7688 Align: kShadowTLSAlignment);
7689 Constant *Cst = dyn_cast<Constant>(Val: ArgShadow);
7690 if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
7691 IRB.CreateStore(Val: getOrigin(V: A),
7692 Ptr: getOriginPtrForArgument(IRB, ArgOffset));
7693 }
7694 }
7695 assert(Store != nullptr);
7696 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
7697 }
7698 assert(Size != 0);
7699 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
7700 }
7701 LLVM_DEBUG(dbgs() << " done with call args\n");
7702
7703 FunctionType *FT = CB.getFunctionType();
7704 if (FT->isVarArg()) {
7705 VAHelper->visitCallBase(CB, IRB);
7706 }
7707
7708 // Now, get the shadow for the RetVal.
7709 if (!CB.getType()->isSized())
7710 return;
7711 // Don't emit the epilogue for musttail call returns.
7712 if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
7713 return;
7714
7715 if (MayCheckCall && CB.hasRetAttr(Kind: Attribute::NoUndef)) {
7716 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7717 setOrigin(V: &CB, Origin: getCleanOrigin());
7718 return;
7719 }
7720
7721 IRBuilder<> IRBBefore(&CB);
7722 // Until we have full dynamic coverage, make sure the retval shadow is 0.
7723 Value *Base = getShadowPtrForRetval(IRB&: IRBBefore);
7724 IRBBefore.CreateAlignedStore(Val: getCleanShadow(V: &CB), Ptr: Base,
7725 Align: kShadowTLSAlignment);
7726 BasicBlock::iterator NextInsn;
7727 if (isa<CallInst>(Val: CB)) {
7728 NextInsn = ++CB.getIterator();
7729 assert(NextInsn != CB.getParent()->end());
7730 } else {
7731 BasicBlock *NormalDest = cast<InvokeInst>(Val&: CB).getNormalDest();
7732 if (!NormalDest->getSinglePredecessor()) {
7733 // FIXME: this case is tricky, so we are just conservative here.
7734 // Perhaps we need to split the edge between this BB and NormalDest,
7735 // but a naive attempt to use SplitEdge leads to a crash.
7736 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7737 setOrigin(V: &CB, Origin: getCleanOrigin());
7738 return;
7739 }
7740 // FIXME: NextInsn is likely in a basic block that has not been visited
7741 // yet. Anything inserted there will be instrumented by MSan later!
7742 NextInsn = NormalDest->getFirstInsertionPt();
7743 assert(NextInsn != NormalDest->end() &&
7744 "Could not find insertion point for retval shadow load");
7745 }
7746 IRBuilder<> IRBAfter(&*NextInsn);
7747 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
7748 Ty: getShadowTy(V: &CB), Ptr: getShadowPtrForRetval(IRB&: IRBAfter), Align: kShadowTLSAlignment,
7749 Name: "_msret");
7750 setShadow(V: &CB, SV: RetvalShadow);
7751 if (MS.TrackOrigins)
7752 setOrigin(V: &CB, Origin: IRBAfter.CreateLoad(Ty: MS.OriginTy, Ptr: getOriginPtrForRetval()));
7753 }
7754
7755 bool isAMustTailRetVal(Value *RetVal) {
7756 if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
7757 RetVal = I->getOperand(i_nocapture: 0);
7758 }
7759 if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
7760 return I->isMustTailCall();
7761 }
7762 return false;
7763 }
7764
7765 void visitReturnInst(ReturnInst &I) {
7766 IRBuilder<> IRB(&I);
7767 Value *RetVal = I.getReturnValue();
7768 if (!RetVal)
7769 return;
7770 // Don't emit the epilogue for musttail call returns.
7771 if (isAMustTailRetVal(RetVal))
7772 return;
7773 Value *ShadowPtr = getShadowPtrForRetval(IRB);
7774 bool HasNoUndef = F.hasRetAttribute(Kind: Attribute::NoUndef);
7775 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
7776 // FIXME: Consider using SpecialCaseList to specify a list of functions that
7777 // must always return fully initialized values. For now, we hardcode "main".
7778 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
7779
7780 Value *Shadow = getShadow(V: RetVal);
7781 bool StoreOrigin = true;
7782 if (EagerCheck) {
7783 insertCheckShadowOf(Val: RetVal, OrigIns: &I);
7784 Shadow = getCleanShadow(V: RetVal);
7785 StoreOrigin = false;
7786 }
7787
7788 // The caller may still expect information passed over TLS if we pass our
7789 // check
7790 if (StoreShadow) {
7791 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: kShadowTLSAlignment);
7792 if (MS.TrackOrigins && StoreOrigin)
7793 IRB.CreateStore(Val: getOrigin(V: RetVal), Ptr: getOriginPtrForRetval());
7794 }
7795 }
7796
7797 void visitPHINode(PHINode &I) {
7798 IRBuilder<> IRB(&I);
7799 if (!PropagateShadow) {
7800 setShadow(V: &I, SV: getCleanShadow(V: &I));
7801 setOrigin(V: &I, Origin: getCleanOrigin());
7802 return;
7803 }
7804
7805 ShadowPHINodes.push_back(Elt: &I);
7806 setShadow(V: &I, SV: IRB.CreatePHI(Ty: getShadowTy(V: &I), NumReservedValues: I.getNumIncomingValues(),
7807 Name: "_msphi_s"));
7808 if (MS.TrackOrigins)
7809 setOrigin(
7810 V: &I, Origin: IRB.CreatePHI(Ty: MS.OriginTy, NumReservedValues: I.getNumIncomingValues(), Name: "_msphi_o"));
7811 }
7812
7813 Value *getLocalVarIdptr(AllocaInst &I) {
7814 ConstantInt *IntConst =
7815 ConstantInt::get(Ty: Type::getInt32Ty(C&: (*F.getParent()).getContext()), V: 0);
7816 return new GlobalVariable(*F.getParent(), IntConst->getType(),
7817 /*isConstant=*/false, GlobalValue::PrivateLinkage,
7818 IntConst);
7819 }
7820
7821 Value *getLocalVarDescription(AllocaInst &I) {
7822 return createPrivateConstGlobalForString(M&: *F.getParent(), Str: I.getName());
7823 }
7824
7825 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7826 if (PoisonStack && ClPoisonStackWithCall) {
7827 IRB.CreateCall(Callee: MS.MsanPoisonStackFn, Args: {&I, Len});
7828 } else {
7829 Value *ShadowBase, *OriginBase;
7830 std::tie(args&: ShadowBase, args&: OriginBase) = getShadowOriginPtr(
7831 Addr: &I, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1), /*isStore*/ true);
7832
7833 Value *PoisonValue = IRB.getInt8(C: PoisonStack ? ClPoisonStackPattern : 0);
7834 IRB.CreateMemSet(Ptr: ShadowBase, Val: PoisonValue, Size: Len, Align: I.getAlign());
7835 }
7836
7837 if (PoisonStack && MS.TrackOrigins) {
7838 Value *Idptr = getLocalVarIdptr(I);
7839 if (ClPrintStackNames) {
7840 Value *Descr = getLocalVarDescription(I);
7841 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginWithDescriptionFn,
7842 Args: {&I, Len, Idptr, Descr});
7843 } else {
7844 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginNoDescriptionFn, Args: {&I, Len, Idptr});
7845 }
7846 }
7847 }
7848
7849 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7850 Value *Descr = getLocalVarDescription(I);
7851 if (PoisonStack) {
7852 IRB.CreateCall(Callee: MS.MsanPoisonAllocaFn, Args: {&I, Len, Descr});
7853 } else {
7854 IRB.CreateCall(Callee: MS.MsanUnpoisonAllocaFn, Args: {&I, Len});
7855 }
7856 }
7857
7858 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
7859 if (!InsPoint)
7860 InsPoint = &I;
7861 NextNodeIRBuilder IRB(InsPoint);
7862 Value *Len = IRB.CreateAllocationSize(DestTy: MS.IntptrTy, AI: &I);
7863
7864 if (MS.CompileKernel)
7865 poisonAllocaKmsan(I, IRB, Len);
7866 else
7867 poisonAllocaUserspace(I, IRB, Len);
7868 }
7869
7870 void visitAllocaInst(AllocaInst &I) {
7871 setShadow(V: &I, SV: getCleanShadow(V: &I));
7872 setOrigin(V: &I, Origin: getCleanOrigin());
7873 // We'll get to this alloca later unless it's poisoned at the corresponding
7874 // llvm.lifetime.start.
7875 AllocaSet.insert(X: &I);
7876 }
7877
7878 void visitSelectInst(SelectInst &I) {
7879 // a = select b, c, d
7880 Value *B = I.getCondition();
7881 Value *C = I.getTrueValue();
7882 Value *D = I.getFalseValue();
7883
7884 handleSelectLikeInst(I, B, C, D);
7885 }
7886
7887 void handleSelectLikeInst(Instruction &I, Value *B, Value *C, Value *D) {
7888 IRBuilder<> IRB(&I);
7889
7890 Value *Sb = getShadow(V: B);
7891 Value *Sc = getShadow(V: C);
7892 Value *Sd = getShadow(V: D);
7893
7894 Value *Ob = MS.TrackOrigins ? getOrigin(V: B) : nullptr;
7895 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
7896 Value *Od = MS.TrackOrigins ? getOrigin(V: D) : nullptr;
7897
7898 // Result shadow if condition shadow is 0.
7899 Value *Sa0 = IRB.CreateSelect(C: B, True: Sc, False: Sd);
7900 Value *Sa1;
7901 if (I.getType()->isAggregateType()) {
7902 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
7903 // an extra "select". This results in much more compact IR.
7904 // Sa = select Sb, poisoned, (select b, Sc, Sd)
7905 Sa1 = getPoisonedShadow(ShadowTy: getShadowTy(OrigTy: I.getType()));
7906 } else if (isScalableNonVectorType(Ty: I.getType())) {
7907 // This is intended to handle target("aarch64.svcount"), which can't be
7908 // handled in the else branch because of incompatibility with CreateXor
7909 // ("The supported LLVM operations on this type are limited to load,
7910 // store, phi, select and alloca instructions").
7911
7912 // TODO: this currently underapproximates. Use Arm SVE EOR in the else
7913 // branch as needed instead.
7914 Sa1 = getCleanShadow(OrigTy: getShadowTy(OrigTy: I.getType()));
7915 } else {
7916 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
7917 // If Sb (condition is poisoned), look for bits in c and d that are equal
7918 // and both unpoisoned.
7919 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
7920
7921 // Cast arguments to shadow-compatible type.
7922 C = CreateAppToShadowCast(IRB, V: C);
7923 D = CreateAppToShadowCast(IRB, V: D);
7924
7925 // Result shadow if condition shadow is 1.
7926 Sa1 = IRB.CreateOr(Ops: {IRB.CreateXor(LHS: C, RHS: D), Sc, Sd});
7927 }
7928 Value *Sa = IRB.CreateSelect(C: Sb, True: Sa1, False: Sa0, Name: "_msprop_select");
7929 setShadow(V: &I, SV: Sa);
7930 if (MS.TrackOrigins) {
7931 // Origins are always i32, so any vector conditions must be flattened.
7932 // FIXME: consider tracking vector origins for app vectors?
7933 if (B->getType()->isVectorTy()) {
7934 B = convertToBool(V: B, IRB);
7935 Sb = convertToBool(V: Sb, IRB);
7936 }
7937 // a = select b, c, d
7938 // Oa = Sb ? Ob : (b ? Oc : Od)
7939 setOrigin(V: &I, Origin: IRB.CreateSelect(C: Sb, True: Ob, False: IRB.CreateSelect(C: B, True: Oc, False: Od)));
7940 }
7941 }
7942
7943 void visitLandingPadInst(LandingPadInst &I) {
7944 // Do nothing.
7945 // See https://github.com/google/sanitizers/issues/504
7946 setShadow(V: &I, SV: getCleanShadow(V: &I));
7947 setOrigin(V: &I, Origin: getCleanOrigin());
7948 }
7949
7950 void visitCatchSwitchInst(CatchSwitchInst &I) {
7951 setShadow(V: &I, SV: getCleanShadow(V: &I));
7952 setOrigin(V: &I, Origin: getCleanOrigin());
7953 }
7954
7955 void visitFuncletPadInst(FuncletPadInst &I) {
7956 setShadow(V: &I, SV: getCleanShadow(V: &I));
7957 setOrigin(V: &I, Origin: getCleanOrigin());
7958 }
7959
7960 void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
7961
7962 void visitExtractValueInst(ExtractValueInst &I) {
7963 IRBuilder<> IRB(&I);
7964 Value *Agg = I.getAggregateOperand();
7965 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
7966 Value *AggShadow = getShadow(V: Agg);
7967 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
7968 Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
7969 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
7970 setShadow(V: &I, SV: ResShadow);
7971 setOriginForNaryOp(I);
7972 }
7973
7974 void visitInsertValueInst(InsertValueInst &I) {
7975 IRBuilder<> IRB(&I);
7976 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
7977 Value *AggShadow = getShadow(V: I.getAggregateOperand());
7978 Value *InsShadow = getShadow(V: I.getInsertedValueOperand());
7979 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
7980 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
7981 Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
7982 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
7983 setShadow(V: &I, SV: Res);
7984 setOriginForNaryOp(I);
7985 }
7986
7987 void dumpInst(Instruction &I, const Twine &Prefix) {
7988 // Instruction name only
7989 // For intrinsics, the full/overloaded name is used
7990 //
7991 // e.g., "call llvm.aarch64.neon.uqsub.v16i8"
7992 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
7993 errs() << "ZZZ:" << Prefix << " call "
7994 << CI->getCalledFunction()->getName() << "\n";
7995 } else {
7996 errs() << "ZZZ:" << Prefix << " " << I.getOpcodeName() << "\n";
7997 }
7998
7999 // Instruction prototype (including return type and parameter types)
8000 // For intrinsics, we use the base/non-overloaded name
8001 //
8002 // e.g., "call <16 x i8> @llvm.aarch64.neon.uqsub(<16 x i8>, <16 x i8>)"
8003 unsigned NumOperands = I.getNumOperands();
8004 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
8005 errs() << "YYY:" << Prefix << " call " << *I.getType() << " @";
8006
8007 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI))
8008 errs() << Intrinsic::getBaseName(id: II->getIntrinsicID());
8009 else
8010 errs() << CI->getCalledFunction()->getName();
8011
8012 errs() << "(";
8013
8014 // The last operand of a CallInst is the function itself.
8015 NumOperands--;
8016 } else
8017 errs() << "YYY:" << Prefix << " " << *I.getType() << " "
8018 << I.getOpcodeName() << "(";
8019
8020 for (size_t i = 0; i < NumOperands; i++) {
8021 if (i > 0)
8022 errs() << ", ";
8023
8024 errs() << *(I.getOperand(i)->getType());
8025 }
8026
8027 errs() << ")\n";
8028
8029 // Full instruction, including types and operand values
8030 // For intrinsics, the full/overloaded name is used
8031 //
8032 // e.g., "%vqsubq_v.i15 = call noundef <16 x i8>
8033 // @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %vext21.i,
8034 // <16 x i8> splat (i8 1)), !dbg !66"
8035 errs() << "QQQ:" << Prefix << " " << I << "\n";
8036 }
8037
8038 void visitResumeInst(ResumeInst &I) {
8039 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
8040 // Nothing to do here.
8041 }
8042
8043 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
8044 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
8045 // Nothing to do here.
8046 }
8047
8048 void visitCatchReturnInst(CatchReturnInst &CRI) {
8049 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
8050 // Nothing to do here.
8051 }
8052
8053 void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
8054 IRBuilder<> &IRB, const DataLayout &DL,
8055 bool isOutput) {
8056 // For each assembly argument, we check its value for being initialized.
8057 // If the argument is a pointer, we assume it points to a single element
8058 // of the corresponding type (or to a 8-byte word, if the type is unsized).
8059 // Each such pointer is instrumented with a call to the runtime library.
8060 Type *OpType = Operand->getType();
8061 // Check the operand value itself.
8062 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8063 if (!OpType->isPointerTy() || !isOutput) {
8064 assert(!isOutput);
8065 return;
8066 }
8067 if (!ElemTy->isSized())
8068 return;
8069 auto Size = DL.getTypeStoreSize(Ty: ElemTy);
8070 Value *SizeVal = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size);
8071 if (MS.CompileKernel) {
8072 IRB.CreateCall(Callee: MS.MsanInstrumentAsmStoreFn, Args: {Operand, SizeVal});
8073 } else {
8074 // ElemTy, derived from elementtype(), does not encode the alignment of
8075 // the pointer. Conservatively assume that the shadow memory is unaligned.
8076 // When Size is large, avoid StoreInst as it would expand to many
8077 // instructions.
8078 auto [ShadowPtr, _] =
8079 getShadowOriginPtrUserspace(Addr: Operand, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1));
8080 if (Size <= 32)
8081 IRB.CreateAlignedStore(Val: getCleanShadow(OrigTy: ElemTy), Ptr: ShadowPtr, Align: Align(1));
8082 else
8083 IRB.CreateMemSet(Ptr: ShadowPtr, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8084 Size: SizeVal, Align: Align(1));
8085 }
8086 }
8087
8088 /// Get the number of output arguments returned by pointers.
8089 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
8090 int NumRetOutputs = 0;
8091 int NumOutputs = 0;
8092 Type *RetTy = cast<Value>(Val: CB)->getType();
8093 if (!RetTy->isVoidTy()) {
8094 // Register outputs are returned via the CallInst return value.
8095 auto *ST = dyn_cast<StructType>(Val: RetTy);
8096 if (ST)
8097 NumRetOutputs = ST->getNumElements();
8098 else
8099 NumRetOutputs = 1;
8100 }
8101 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
8102 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
8103 switch (Info.Type) {
8104 case InlineAsm::isOutput:
8105 NumOutputs++;
8106 break;
8107 default:
8108 break;
8109 }
8110 }
8111 return NumOutputs - NumRetOutputs;
8112 }
8113
8114 void visitAsmInstruction(Instruction &I) {
8115 // Conservative inline assembly handling: check for poisoned shadow of
8116 // asm() arguments, then unpoison the result and all the memory locations
8117 // pointed to by those arguments.
8118 // An inline asm() statement in C++ contains lists of input and output
8119 // arguments used by the assembly code. These are mapped to operands of the
8120 // CallInst as follows:
8121 // - nR register outputs ("=r) are returned by value in a single structure
8122 // (SSA value of the CallInst);
8123 // - nO other outputs ("=m" and others) are returned by pointer as first
8124 // nO operands of the CallInst;
8125 // - nI inputs ("r", "m" and others) are passed to CallInst as the
8126 // remaining nI operands.
8127 // The total number of asm() arguments in the source is nR+nO+nI, and the
8128 // corresponding CallInst has nO+nI+1 operands (the last operand is the
8129 // function to be called).
8130 const DataLayout &DL = F.getDataLayout();
8131 CallBase *CB = cast<CallBase>(Val: &I);
8132 IRBuilder<> IRB(&I);
8133 InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8134 int OutputArgs = getNumOutputArgs(IA, CB);
8135 // The last operand of a CallInst is the function itself.
8136 int NumOperands = CB->getNumOperands() - 1;
8137
8138 // Check input arguments. Doing so before unpoisoning output arguments, so
8139 // that we won't overwrite uninit values before checking them.
8140 for (int i = OutputArgs; i < NumOperands; i++) {
8141 Value *Operand = CB->getOperand(i_nocapture: i);
8142 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8143 /*isOutput*/ false);
8144 }
8145 // Unpoison output arguments. This must happen before the actual InlineAsm
8146 // call, so that the shadow for memory published in the asm() statement
8147 // remains valid.
8148 for (int i = 0; i < OutputArgs; i++) {
8149 Value *Operand = CB->getOperand(i_nocapture: i);
8150 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8151 /*isOutput*/ true);
8152 }
8153
8154 setShadow(V: &I, SV: getCleanShadow(V: &I));
8155 setOrigin(V: &I, Origin: getCleanOrigin());
8156 }
8157
8158 void visitFreezeInst(FreezeInst &I) {
8159 // Freeze always returns a fully defined value.
8160 setShadow(V: &I, SV: getCleanShadow(V: &I));
8161 setOrigin(V: &I, Origin: getCleanOrigin());
8162 }
8163
8164 void visitInstruction(Instruction &I) {
8165 // Everything else: stop propagating and check for poisoned shadow.
8166 if (ClDumpStrictInstructions)
8167 dumpInst(I, Prefix: "Strict");
8168 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
8169 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
8170 Value *Operand = I.getOperand(i);
8171 if (Operand->getType()->isSized())
8172 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8173 }
8174 setShadow(V: &I, SV: getCleanShadow(V: &I));
8175 setOrigin(V: &I, Origin: getCleanOrigin());
8176 }
8177};
8178
8179struct VarArgHelperBase : public VarArgHelper {
8180 Function &F;
8181 MemorySanitizer &MS;
8182 MemorySanitizerVisitor &MSV;
8183 SmallVector<CallInst *, 16> VAStartInstrumentationList;
8184 const unsigned VAListTagSize;
8185
8186 VarArgHelperBase(Function &F, MemorySanitizer &MS,
8187 MemorySanitizerVisitor &MSV, unsigned VAListTagSize)
8188 : F(F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
8189
8190 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8191 Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
8192 return IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
8193 }
8194
8195 /// Compute the shadow address for a given va_arg.
8196 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8197 return IRB.CreatePtrAdd(
8198 Ptr: MS.VAArgTLS, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg_va_s");
8199 }
8200
8201 /// Compute the shadow address for a given va_arg.
8202 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset,
8203 unsigned ArgSize) {
8204 // Make sure we don't overflow __msan_va_arg_tls.
8205 if (ArgOffset + ArgSize > kParamTLSSize)
8206 return nullptr;
8207 return getShadowPtrForVAArgument(IRB, ArgOffset);
8208 }
8209
8210 /// Compute the origin address for a given va_arg.
8211 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
8212 // getOriginPtrForVAArgument() is always called after
8213 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
8214 // overflow.
8215 return IRB.CreatePtrAdd(Ptr: MS.VAArgOriginTLS,
8216 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
8217 Name: "_msarg_va_o");
8218 }
8219
8220 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
8221 unsigned BaseOffset) {
8222 // The tails of __msan_va_arg_tls is not large enough to fit full
8223 // value shadow, but it will be copied to backup anyway. Make it
8224 // clean.
8225 if (BaseOffset >= kParamTLSSize)
8226 return;
8227 Value *TailSize =
8228 ConstantInt::getSigned(Ty: IRB.getInt32Ty(), V: kParamTLSSize - BaseOffset);
8229 IRB.CreateMemSet(Ptr: ShadowBase, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8230 Size: TailSize, Align: Align(8));
8231 }
8232
8233 void unpoisonVAListTagForInst(IntrinsicInst &I) {
8234 IRBuilder<> IRB(&I);
8235 Value *VAListTag = I.getArgOperand(i: 0);
8236 const Align Alignment = Align(8);
8237 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
8238 Addr: VAListTag, IRB, ShadowTy: IRB.getInt8Ty(), Alignment, /*isStore*/ true);
8239 // Unpoison the whole __va_list_tag.
8240 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8241 Size: VAListTagSize, Align: Alignment, isVolatile: false);
8242 }
8243
8244 void visitVAStartInst(VAStartInst &I) override {
8245 if (F.getCallingConv() == CallingConv::Win64)
8246 return;
8247 VAStartInstrumentationList.push_back(Elt: &I);
8248 unpoisonVAListTagForInst(I);
8249 }
8250
8251 void visitVACopyInst(VACopyInst &I) override {
8252 if (F.getCallingConv() == CallingConv::Win64)
8253 return;
8254 unpoisonVAListTagForInst(I);
8255 }
8256};
8257
8258/// AMD64-specific implementation of VarArgHelper.
8259struct VarArgAMD64Helper : public VarArgHelperBase {
8260 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
8261 // See a comment in visitCallBase for more details.
8262 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
8263 static const unsigned AMD64FpEndOffsetSSE = 176;
8264 // If SSE is disabled, fp_offset in va_list is zero.
8265 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
8266
8267 unsigned AMD64FpEndOffset;
8268 AllocaInst *VAArgTLSCopy = nullptr;
8269 AllocaInst *VAArgTLSOriginCopy = nullptr;
8270 Value *VAArgOverflowSize = nullptr;
8271
8272 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8273
8274 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
8275 MemorySanitizerVisitor &MSV)
8276 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/24) {
8277 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
8278 for (const auto &Attr : F.getAttributes().getFnAttrs()) {
8279 if (Attr.isStringAttribute() &&
8280 (Attr.getKindAsString() == "target-features")) {
8281 if (Attr.getValueAsString().contains(Other: "-sse"))
8282 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
8283 break;
8284 }
8285 }
8286 }
8287
8288 ArgKind classifyArgument(Value *arg) {
8289 // A very rough approximation of X86_64 argument classification rules.
8290 Type *T = arg->getType();
8291 if (T->isX86_FP80Ty())
8292 return AK_Memory;
8293 if (T->isFPOrFPVectorTy())
8294 return AK_FloatingPoint;
8295 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
8296 return AK_GeneralPurpose;
8297 if (T->isPointerTy())
8298 return AK_GeneralPurpose;
8299 return AK_Memory;
8300 }
8301
8302 // For VarArg functions, store the argument shadow in an ABI-specific format
8303 // that corresponds to va_list layout.
8304 // We do this because Clang lowers va_arg in the frontend, and this pass
8305 // only sees the low level code that deals with va_list internals.
8306 // A much easier alternative (provided that Clang emits va_arg instructions)
8307 // would have been to associate each live instance of va_list with a copy of
8308 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
8309 // order.
8310 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8311 unsigned GpOffset = 0;
8312 unsigned FpOffset = AMD64GpEndOffset;
8313 unsigned OverflowOffset = AMD64FpEndOffset;
8314 const DataLayout &DL = F.getDataLayout();
8315
8316 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8317 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8318 bool IsByVal = CB.isByValArgument(ArgNo);
8319 if (IsByVal) {
8320 // ByVal arguments always go to the overflow area.
8321 // Fixed arguments passed through the overflow area will be stepped
8322 // over by va_start, so don't count them towards the offset.
8323 if (IsFixed)
8324 continue;
8325 assert(A->getType()->isPointerTy());
8326 Type *RealTy = CB.getParamByValType(ArgNo);
8327 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8328 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8329 unsigned BaseOffset = OverflowOffset;
8330 Value *ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8331 Value *OriginBase = nullptr;
8332 if (MS.TrackOrigins)
8333 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8334 OverflowOffset += AlignedSize;
8335
8336 if (OverflowOffset > kParamTLSSize) {
8337 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8338 continue; // We have no space to copy shadow there.
8339 }
8340
8341 Value *ShadowPtr, *OriginPtr;
8342 std::tie(args&: ShadowPtr, args&: OriginPtr) =
8343 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: kShadowTLSAlignment,
8344 /*isStore*/ false);
8345 IRB.CreateMemCpy(Dst: ShadowBase, DstAlign: kShadowTLSAlignment, Src: ShadowPtr,
8346 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8347 if (MS.TrackOrigins)
8348 IRB.CreateMemCpy(Dst: OriginBase, DstAlign: kShadowTLSAlignment, Src: OriginPtr,
8349 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8350 } else {
8351 ArgKind AK = classifyArgument(arg: A);
8352 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
8353 AK = AK_Memory;
8354 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
8355 AK = AK_Memory;
8356 Value *ShadowBase, *OriginBase = nullptr;
8357 switch (AK) {
8358 case AK_GeneralPurpose:
8359 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: GpOffset);
8360 if (MS.TrackOrigins)
8361 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset);
8362 GpOffset += 8;
8363 assert(GpOffset <= kParamTLSSize);
8364 break;
8365 case AK_FloatingPoint:
8366 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: FpOffset);
8367 if (MS.TrackOrigins)
8368 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
8369 FpOffset += 16;
8370 assert(FpOffset <= kParamTLSSize);
8371 break;
8372 case AK_Memory:
8373 if (IsFixed)
8374 continue;
8375 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8376 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8377 unsigned BaseOffset = OverflowOffset;
8378 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8379 if (MS.TrackOrigins) {
8380 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8381 }
8382 OverflowOffset += AlignedSize;
8383 if (OverflowOffset > kParamTLSSize) {
8384 // We have no space to copy shadow there.
8385 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8386 continue;
8387 }
8388 }
8389 // Take fixed arguments into account for GpOffset and FpOffset,
8390 // but don't actually store shadows for them.
8391 // TODO(glider): don't call get*PtrForVAArgument() for them.
8392 if (IsFixed)
8393 continue;
8394 Value *Shadow = MSV.getShadow(V: A);
8395 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowBase, Align: kShadowTLSAlignment);
8396 if (MS.TrackOrigins) {
8397 Value *Origin = MSV.getOrigin(V: A);
8398 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
8399 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
8400 Alignment: std::max(a: kShadowTLSAlignment, b: kMinOriginAlignment));
8401 }
8402 }
8403 }
8404 Constant *OverflowSize =
8405 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AMD64FpEndOffset);
8406 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8407 }
8408
8409 void finalizeInstrumentation() override {
8410 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8411 "finalizeInstrumentation called twice");
8412 if (!VAStartInstrumentationList.empty()) {
8413 // If there is a va_start in this function, make a backup copy of
8414 // va_arg_tls somewhere in the function entry block.
8415 IRBuilder<> IRB(MSV.FnPrologueEnd);
8416 VAArgOverflowSize =
8417 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8418 Value *CopySize = IRB.CreateAdd(
8419 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AMD64FpEndOffset), RHS: VAArgOverflowSize);
8420 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8421 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8422 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8423 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8424
8425 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8426 ID: Intrinsic::umin, LHS: CopySize,
8427 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8428 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8429 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8430 if (MS.TrackOrigins) {
8431 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8432 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
8433 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
8434 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8435 }
8436 }
8437
8438 // Instrument va_start.
8439 // Copy va_list shadow from the backup copy of the TLS contents.
8440 for (CallInst *OrigInst : VAStartInstrumentationList) {
8441 NextNodeIRBuilder IRB(OrigInst);
8442 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8443
8444 Value *RegSaveAreaPtrPtr =
8445 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 16));
8446 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8447 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8448 const Align Alignment = Align(16);
8449 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8450 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8451 Alignment, /*isStore*/ true);
8452 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8453 Size: AMD64FpEndOffset);
8454 if (MS.TrackOrigins)
8455 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
8456 SrcAlign: Alignment, Size: AMD64FpEndOffset);
8457 Value *OverflowArgAreaPtrPtr =
8458 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
8459 Value *OverflowArgAreaPtr =
8460 IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
8461 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
8462 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
8463 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8464 Alignment, /*isStore*/ true);
8465 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
8466 Idx0: AMD64FpEndOffset);
8467 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8468 Size: VAArgOverflowSize);
8469 if (MS.TrackOrigins) {
8470 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
8471 Idx0: AMD64FpEndOffset);
8472 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8473 Size: VAArgOverflowSize);
8474 }
8475 }
8476 }
8477};
8478
8479/// AArch64-specific implementation of VarArgHelper.
8480struct VarArgAArch64Helper : public VarArgHelperBase {
8481 static const unsigned kAArch64GrArgSize = 64;
8482 static const unsigned kAArch64VrArgSize = 128;
8483
8484 static const unsigned AArch64GrBegOffset = 0;
8485 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
8486 // Make VR space aligned to 16 bytes.
8487 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
8488 static const unsigned AArch64VrEndOffset =
8489 AArch64VrBegOffset + kAArch64VrArgSize;
8490 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
8491
8492 AllocaInst *VAArgTLSCopy = nullptr;
8493 Value *VAArgOverflowSize = nullptr;
8494
8495 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8496
8497 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
8498 MemorySanitizerVisitor &MSV)
8499 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/32) {}
8500
8501 // A very rough approximation of aarch64 argument classification rules.
8502 std::pair<ArgKind, uint64_t> classifyArgument(Type *T) {
8503 if (T->isIntOrPtrTy() && T->getPrimitiveSizeInBits() <= 64)
8504 return {AK_GeneralPurpose, 1};
8505 if (T->isFloatingPointTy() && T->getPrimitiveSizeInBits() <= 128)
8506 return {AK_FloatingPoint, 1};
8507
8508 if (T->isArrayTy()) {
8509 auto R = classifyArgument(T: T->getArrayElementType());
8510 R.second *= T->getScalarType()->getArrayNumElements();
8511 return R;
8512 }
8513
8514 if (const FixedVectorType *FV = dyn_cast<FixedVectorType>(Val: T)) {
8515 auto R = classifyArgument(T: FV->getScalarType());
8516 R.second *= FV->getNumElements();
8517 return R;
8518 }
8519
8520 LLVM_DEBUG(errs() << "Unknown vararg type: " << *T << "\n");
8521 return {AK_Memory, 0};
8522 }
8523
8524 // The instrumentation stores the argument shadow in a non ABI-specific
8525 // format because it does not know which argument is named (since Clang,
8526 // like x86_64 case, lowers the va_args in the frontend and this pass only
8527 // sees the low level code that deals with va_list internals).
8528 // The first seven GR registers are saved in the first 56 bytes of the
8529 // va_arg tls arra, followed by the first 8 FP/SIMD registers, and then
8530 // the remaining arguments.
8531 // Using constant offset within the va_arg TLS array allows fast copy
8532 // in the finalize instrumentation.
8533 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8534 unsigned GrOffset = AArch64GrBegOffset;
8535 unsigned VrOffset = AArch64VrBegOffset;
8536 unsigned OverflowOffset = AArch64VAEndOffset;
8537
8538 const DataLayout &DL = F.getDataLayout();
8539 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8540 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8541 auto [AK, RegNum] = classifyArgument(T: A->getType());
8542 if (AK == AK_GeneralPurpose &&
8543 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
8544 AK = AK_Memory;
8545 if (AK == AK_FloatingPoint &&
8546 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
8547 AK = AK_Memory;
8548 Value *Base;
8549 switch (AK) {
8550 case AK_GeneralPurpose:
8551 Base = getShadowPtrForVAArgument(IRB, ArgOffset: GrOffset);
8552 GrOffset += 8 * RegNum;
8553 break;
8554 case AK_FloatingPoint:
8555 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VrOffset);
8556 VrOffset += 16 * RegNum;
8557 break;
8558 case AK_Memory:
8559 // Don't count fixed arguments in the overflow area - va_start will
8560 // skip right over them.
8561 if (IsFixed)
8562 continue;
8563 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8564 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8565 unsigned BaseOffset = OverflowOffset;
8566 Base = getShadowPtrForVAArgument(IRB, ArgOffset: BaseOffset);
8567 OverflowOffset += AlignedSize;
8568 if (OverflowOffset > kParamTLSSize) {
8569 // We have no space to copy shadow there.
8570 CleanUnusedTLS(IRB, ShadowBase: Base, BaseOffset);
8571 continue;
8572 }
8573 break;
8574 }
8575 // Count Gp/Vr fixed arguments to their respective offsets, but don't
8576 // bother to actually store a shadow.
8577 if (IsFixed)
8578 continue;
8579 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8580 }
8581 Constant *OverflowSize =
8582 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AArch64VAEndOffset);
8583 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8584 }
8585
8586 // Retrieve a va_list field of 'void*' size.
8587 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8588 Value *SaveAreaPtrPtr =
8589 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8590 return IRB.CreateLoad(Ty: Type::getInt64Ty(C&: *MS.C), Ptr: SaveAreaPtrPtr);
8591 }
8592
8593 // Retrieve a va_list field of 'int' size.
8594 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8595 Value *SaveAreaPtr =
8596 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8597 Value *SaveArea32 = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: SaveAreaPtr);
8598 return IRB.CreateSExt(V: SaveArea32, DestTy: MS.IntptrTy);
8599 }
8600
8601 void finalizeInstrumentation() override {
8602 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8603 "finalizeInstrumentation called twice");
8604 if (!VAStartInstrumentationList.empty()) {
8605 // If there is a va_start in this function, make a backup copy of
8606 // va_arg_tls somewhere in the function entry block.
8607 IRBuilder<> IRB(MSV.FnPrologueEnd);
8608 VAArgOverflowSize =
8609 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8610 Value *CopySize = IRB.CreateAdd(
8611 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AArch64VAEndOffset), RHS: VAArgOverflowSize);
8612 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8613 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8614 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8615 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8616
8617 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8618 ID: Intrinsic::umin, LHS: CopySize,
8619 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8620 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8621 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8622 }
8623
8624 Value *GrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64GrArgSize);
8625 Value *VrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64VrArgSize);
8626
8627 // Instrument va_start, copy va_list shadow from the backup copy of
8628 // the TLS contents.
8629 for (CallInst *OrigInst : VAStartInstrumentationList) {
8630 NextNodeIRBuilder IRB(OrigInst);
8631
8632 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8633
8634 // The variadic ABI for AArch64 creates two areas to save the incoming
8635 // argument registers (one for 64-bit general register xn-x7 and another
8636 // for 128-bit FP/SIMD vn-v7).
8637 // We need then to propagate the shadow arguments on both regions
8638 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
8639 // The remaining arguments are saved on shadow for 'va::stack'.
8640 // One caveat is it requires only to propagate the non-named arguments,
8641 // however on the call site instrumentation 'all' the arguments are
8642 // saved. So to copy the shadow values from the va_arg TLS array
8643 // we need to adjust the offset for both GR and VR fields based on
8644 // the __{gr,vr}_offs value (since they are stores based on incoming
8645 // named arguments).
8646 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
8647
8648 // Read the stack pointer from the va_list.
8649 Value *StackSaveAreaPtr =
8650 IRB.CreateIntToPtr(V: getVAField64(IRB, VAListTag, offset: 0), DestTy: RegSaveAreaPtrTy);
8651
8652 // Read both the __gr_top and __gr_off and add them up.
8653 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 8);
8654 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, offset: 24);
8655
8656 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
8657 V: IRB.CreateAdd(LHS: GrTopSaveAreaPtr, RHS: GrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8658
8659 // Read both the __vr_top and __vr_off and add them up.
8660 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 16);
8661 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, offset: 28);
8662
8663 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
8664 V: IRB.CreateAdd(LHS: VrTopSaveAreaPtr, RHS: VrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8665
8666 // It does not know how many named arguments is being used and, on the
8667 // callsite all the arguments were saved. Since __gr_off is defined as
8668 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
8669 // argument by ignoring the bytes of shadow from named arguments.
8670 Value *GrRegSaveAreaShadowPtrOff =
8671 IRB.CreateAdd(LHS: GrArgSize, RHS: GrOffSaveArea);
8672
8673 Value *GrRegSaveAreaShadowPtr =
8674 MSV.getShadowOriginPtr(Addr: GrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8675 Alignment: Align(8), /*isStore*/ true)
8676 .first;
8677
8678 Value *GrSrcPtr =
8679 IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy, Offset: GrRegSaveAreaShadowPtrOff);
8680 Value *GrCopySize = IRB.CreateSub(LHS: GrArgSize, RHS: GrRegSaveAreaShadowPtrOff);
8681
8682 IRB.CreateMemCpy(Dst: GrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: GrSrcPtr, SrcAlign: Align(8),
8683 Size: GrCopySize);
8684
8685 // Again, but for FP/SIMD values.
8686 Value *VrRegSaveAreaShadowPtrOff =
8687 IRB.CreateAdd(LHS: VrArgSize, RHS: VrOffSaveArea);
8688
8689 Value *VrRegSaveAreaShadowPtr =
8690 MSV.getShadowOriginPtr(Addr: VrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8691 Alignment: Align(8), /*isStore*/ true)
8692 .first;
8693
8694 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
8695 Ptr: IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy,
8696 Offset: IRB.getInt32(C: AArch64VrBegOffset)),
8697 Offset: VrRegSaveAreaShadowPtrOff);
8698 Value *VrCopySize = IRB.CreateSub(LHS: VrArgSize, RHS: VrRegSaveAreaShadowPtrOff);
8699
8700 IRB.CreateMemCpy(Dst: VrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: VrSrcPtr, SrcAlign: Align(8),
8701 Size: VrCopySize);
8702
8703 // And finally for remaining arguments.
8704 Value *StackSaveAreaShadowPtr =
8705 MSV.getShadowOriginPtr(Addr: StackSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8706 Alignment: Align(16), /*isStore*/ true)
8707 .first;
8708
8709 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
8710 Ptr: VAArgTLSCopy, Offset: IRB.getInt32(C: AArch64VAEndOffset));
8711
8712 IRB.CreateMemCpy(Dst: StackSaveAreaShadowPtr, DstAlign: Align(16), Src: StackSrcPtr,
8713 SrcAlign: Align(16), Size: VAArgOverflowSize);
8714 }
8715 }
8716};
8717
8718/// PowerPC64-specific implementation of VarArgHelper.
8719struct VarArgPowerPC64Helper : public VarArgHelperBase {
8720 AllocaInst *VAArgTLSCopy = nullptr;
8721 Value *VAArgSize = nullptr;
8722
8723 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
8724 MemorySanitizerVisitor &MSV)
8725 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
8726
8727 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8728 // For PowerPC, we need to deal with alignment of stack arguments -
8729 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
8730 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
8731 // For that reason, we compute current offset from stack pointer (which is
8732 // always properly aligned), and offset for the first vararg, then subtract
8733 // them.
8734 unsigned VAArgBase;
8735 Triple TargetTriple(F.getParent()->getTargetTriple());
8736 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
8737 // and 32 bytes for ABIv2. This is usually determined by target
8738 // endianness, but in theory could be overridden by function attribute.
8739 if (TargetTriple.isPPC64ELFv2ABI())
8740 VAArgBase = 32;
8741 else
8742 VAArgBase = 48;
8743 unsigned VAArgOffset = VAArgBase;
8744 const DataLayout &DL = F.getDataLayout();
8745 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8746 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8747 bool IsByVal = CB.isByValArgument(ArgNo);
8748 if (IsByVal) {
8749 assert(A->getType()->isPointerTy());
8750 Type *RealTy = CB.getParamByValType(ArgNo);
8751 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8752 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(8));
8753 if (ArgAlign < 8)
8754 ArgAlign = Align(8);
8755 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8756 if (!IsFixed) {
8757 Value *Base =
8758 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8759 if (Base) {
8760 Value *AShadowPtr, *AOriginPtr;
8761 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8762 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8763 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8764
8765 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8766 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8767 }
8768 }
8769 VAArgOffset += alignTo(Size: ArgSize, A: Align(8));
8770 } else {
8771 Value *Base;
8772 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8773 Align ArgAlign = Align(8);
8774 if (A->getType()->isArrayTy()) {
8775 // Arrays are aligned to element size, except for long double
8776 // arrays, which are aligned to 8 bytes.
8777 Type *ElementTy = A->getType()->getArrayElementType();
8778 if (!ElementTy->isPPC_FP128Ty())
8779 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8780 } else if (A->getType()->isVectorTy()) {
8781 // Vectors are naturally aligned.
8782 ArgAlign = Align(ArgSize);
8783 }
8784 if (ArgAlign < 8)
8785 ArgAlign = Align(8);
8786 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8787 if (DL.isBigEndian()) {
8788 // Adjusting the shadow for argument with size < 8 to match the
8789 // placement of bits in big endian system
8790 if (ArgSize < 8)
8791 VAArgOffset += (8 - ArgSize);
8792 }
8793 if (!IsFixed) {
8794 Base =
8795 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8796 if (Base)
8797 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8798 }
8799 VAArgOffset += ArgSize;
8800 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(8));
8801 }
8802 if (IsFixed)
8803 VAArgBase = VAArgOffset;
8804 }
8805
8806 Constant *TotalVAArgSize =
8807 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8808 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8809 // a new class member i.e. it is the total size of all VarArgs.
8810 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8811 }
8812
8813 void finalizeInstrumentation() override {
8814 assert(!VAArgSize && !VAArgTLSCopy &&
8815 "finalizeInstrumentation called twice");
8816 IRBuilder<> IRB(MSV.FnPrologueEnd);
8817 VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8818 Value *CopySize = VAArgSize;
8819
8820 if (!VAStartInstrumentationList.empty()) {
8821 // If there is a va_start in this function, make a backup copy of
8822 // va_arg_tls somewhere in the function entry block.
8823
8824 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8825 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8826 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8827 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8828
8829 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8830 ID: Intrinsic::umin, LHS: CopySize,
8831 RHS: ConstantInt::get(Ty: IRB.getInt64Ty(), V: kParamTLSSize));
8832 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8833 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8834 }
8835
8836 // Instrument va_start.
8837 // Copy va_list shadow from the backup copy of the TLS contents.
8838 for (CallInst *OrigInst : VAStartInstrumentationList) {
8839 NextNodeIRBuilder IRB(OrigInst);
8840 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8841 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8842
8843 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
8844
8845 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8846 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8847 const DataLayout &DL = F.getDataLayout();
8848 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8849 const Align Alignment = Align(IntptrSize);
8850 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8851 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8852 Alignment, /*isStore*/ true);
8853 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8854 Size: CopySize);
8855 }
8856 }
8857};
8858
8859/// PowerPC32-specific implementation of VarArgHelper.
8860struct VarArgPowerPC32Helper : public VarArgHelperBase {
8861 AllocaInst *VAArgTLSCopy = nullptr;
8862 Value *VAArgSize = nullptr;
8863
8864 VarArgPowerPC32Helper(Function &F, MemorySanitizer &MS,
8865 MemorySanitizerVisitor &MSV)
8866 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/12) {}
8867
8868 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8869 unsigned VAArgBase;
8870 // Parameter save area is 8 bytes from frame pointer in PPC32
8871 VAArgBase = 8;
8872 unsigned VAArgOffset = VAArgBase;
8873 const DataLayout &DL = F.getDataLayout();
8874 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8875 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8876 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8877 bool IsByVal = CB.isByValArgument(ArgNo);
8878 if (IsByVal) {
8879 assert(A->getType()->isPointerTy());
8880 Type *RealTy = CB.getParamByValType(ArgNo);
8881 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8882 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
8883 if (ArgAlign < IntptrSize)
8884 ArgAlign = Align(IntptrSize);
8885 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8886 if (!IsFixed) {
8887 Value *Base =
8888 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8889 if (Base) {
8890 Value *AShadowPtr, *AOriginPtr;
8891 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8892 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8893 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8894
8895 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8896 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8897 }
8898 }
8899 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
8900 } else {
8901 Value *Base;
8902 Type *ArgTy = A->getType();
8903
8904 // On PPC 32 floating point variable arguments are stored in separate
8905 // area: fp_save_area = reg_save_area + 4*8. We do not copy shaodow for
8906 // them as they will be found when checking call arguments.
8907 if (!ArgTy->isFloatingPointTy()) {
8908 uint64_t ArgSize = DL.getTypeAllocSize(Ty: ArgTy);
8909 Align ArgAlign = Align(IntptrSize);
8910 if (ArgTy->isArrayTy()) {
8911 // Arrays are aligned to element size, except for long double
8912 // arrays, which are aligned to 8 bytes.
8913 Type *ElementTy = ArgTy->getArrayElementType();
8914 if (!ElementTy->isPPC_FP128Ty())
8915 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8916 } else if (ArgTy->isVectorTy()) {
8917 // Vectors are naturally aligned.
8918 ArgAlign = Align(ArgSize);
8919 }
8920 if (ArgAlign < IntptrSize)
8921 ArgAlign = Align(IntptrSize);
8922 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8923 if (DL.isBigEndian()) {
8924 // Adjusting the shadow for argument with size < IntptrSize to match
8925 // the placement of bits in big endian system
8926 if (ArgSize < IntptrSize)
8927 VAArgOffset += (IntptrSize - ArgSize);
8928 }
8929 if (!IsFixed) {
8930 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase,
8931 ArgSize);
8932 if (Base)
8933 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base,
8934 Align: kShadowTLSAlignment);
8935 }
8936 VAArgOffset += ArgSize;
8937 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
8938 }
8939 }
8940 }
8941
8942 Constant *TotalVAArgSize =
8943 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8944 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8945 // a new class member i.e. it is the total size of all VarArgs.
8946 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8947 }
8948
8949 void finalizeInstrumentation() override {
8950 assert(!VAArgSize && !VAArgTLSCopy &&
8951 "finalizeInstrumentation called twice");
8952 IRBuilder<> IRB(MSV.FnPrologueEnd);
8953 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
8954 Value *CopySize = VAArgSize;
8955
8956 if (!VAStartInstrumentationList.empty()) {
8957 // If there is a va_start in this function, make a backup copy of
8958 // va_arg_tls somewhere in the function entry block.
8959
8960 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8961 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8962 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8963 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8964
8965 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8966 ID: Intrinsic::umin, LHS: CopySize,
8967 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8968 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8969 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8970 }
8971
8972 // Instrument va_start.
8973 // Copy va_list shadow from the backup copy of the TLS contents.
8974 for (CallInst *OrigInst : VAStartInstrumentationList) {
8975 NextNodeIRBuilder IRB(OrigInst);
8976 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8977 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8978 Value *RegSaveAreaSize = CopySize;
8979
8980 // In PPC32 va_list_tag is a struct
8981 RegSaveAreaPtrPtr =
8982 IRB.CreateAdd(LHS: RegSaveAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
8983
8984 // On PPC 32 reg_save_area can only hold 32 bytes of data
8985 RegSaveAreaSize = IRB.CreateBinaryIntrinsic(
8986 ID: Intrinsic::umin, LHS: CopySize, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
8987
8988 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
8989 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8990
8991 const DataLayout &DL = F.getDataLayout();
8992 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8993 const Align Alignment = Align(IntptrSize);
8994
8995 { // Copy reg save area
8996 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8997 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8998 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8999 Alignment, /*isStore*/ true);
9000 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy,
9001 SrcAlign: Alignment, Size: RegSaveAreaSize);
9002
9003 RegSaveAreaShadowPtr =
9004 IRB.CreatePtrToInt(V: RegSaveAreaShadowPtr, DestTy: MS.IntptrTy);
9005 Value *FPSaveArea = IRB.CreateAdd(LHS: RegSaveAreaShadowPtr,
9006 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
9007 FPSaveArea = IRB.CreateIntToPtr(V: FPSaveArea, DestTy: MS.PtrTy);
9008 // We fill fp shadow with zeroes as uninitialized fp args should have
9009 // been found during call base check
9010 IRB.CreateMemSet(Ptr: FPSaveArea, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
9011 Size: ConstantInt::get(Ty: MS.IntptrTy, V: 32), Align: Alignment);
9012 }
9013
9014 { // Copy overflow area
9015 // RegSaveAreaSize is min(CopySize, 32) -> no overflow can occur
9016 Value *OverflowAreaSize = IRB.CreateSub(LHS: CopySize, RHS: RegSaveAreaSize);
9017
9018 Value *OverflowAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9019 OverflowAreaPtrPtr =
9020 IRB.CreateAdd(LHS: OverflowAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 4));
9021 OverflowAreaPtrPtr = IRB.CreateIntToPtr(V: OverflowAreaPtrPtr, DestTy: MS.PtrTy);
9022
9023 Value *OverflowAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowAreaPtrPtr);
9024
9025 Value *OverflowAreaShadowPtr, *OverflowAreaOriginPtr;
9026 std::tie(args&: OverflowAreaShadowPtr, args&: OverflowAreaOriginPtr) =
9027 MSV.getShadowOriginPtr(Addr: OverflowAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9028 Alignment, /*isStore*/ true);
9029
9030 Value *OverflowVAArgTLSCopyPtr =
9031 IRB.CreatePtrToInt(V: VAArgTLSCopy, DestTy: MS.IntptrTy);
9032 OverflowVAArgTLSCopyPtr =
9033 IRB.CreateAdd(LHS: OverflowVAArgTLSCopyPtr, RHS: RegSaveAreaSize);
9034
9035 OverflowVAArgTLSCopyPtr =
9036 IRB.CreateIntToPtr(V: OverflowVAArgTLSCopyPtr, DestTy: MS.PtrTy);
9037 IRB.CreateMemCpy(Dst: OverflowAreaShadowPtr, DstAlign: Alignment,
9038 Src: OverflowVAArgTLSCopyPtr, SrcAlign: Alignment, Size: OverflowAreaSize);
9039 }
9040 }
9041 }
9042};
9043
9044/// SystemZ-specific implementation of VarArgHelper.
9045struct VarArgSystemZHelper : public VarArgHelperBase {
9046 static const unsigned SystemZGpOffset = 16;
9047 static const unsigned SystemZGpEndOffset = 56;
9048 static const unsigned SystemZFpOffset = 128;
9049 static const unsigned SystemZFpEndOffset = 160;
9050 static const unsigned SystemZMaxVrArgs = 8;
9051 static const unsigned SystemZRegSaveAreaSize = 160;
9052 static const unsigned SystemZOverflowOffset = 160;
9053 static const unsigned SystemZVAListTagSize = 32;
9054 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
9055 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
9056
9057 bool IsSoftFloatABI;
9058 AllocaInst *VAArgTLSCopy = nullptr;
9059 AllocaInst *VAArgTLSOriginCopy = nullptr;
9060 Value *VAArgOverflowSize = nullptr;
9061
9062 enum class ArgKind {
9063 GeneralPurpose,
9064 FloatingPoint,
9065 Vector,
9066 Memory,
9067 Indirect,
9068 };
9069
9070 enum class ShadowExtension { None, Zero, Sign };
9071
9072 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
9073 MemorySanitizerVisitor &MSV)
9074 : VarArgHelperBase(F, MS, MSV, SystemZVAListTagSize),
9075 IsSoftFloatABI(F.getFnAttribute(Kind: "use-soft-float").getValueAsBool()) {}
9076
9077 ArgKind classifyArgument(Type *T) {
9078 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
9079 // only a few possibilities of what it can be. In particular, enums, single
9080 // element structs and large types have already been taken care of.
9081
9082 // Some i128 and fp128 arguments are converted to pointers only in the
9083 // back end.
9084 if (T->isIntegerTy(BitWidth: 128) || T->isFP128Ty())
9085 return ArgKind::Indirect;
9086 if (T->isFloatingPointTy())
9087 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
9088 if (T->isIntegerTy() || T->isPointerTy())
9089 return ArgKind::GeneralPurpose;
9090 if (T->isVectorTy())
9091 return ArgKind::Vector;
9092 return ArgKind::Memory;
9093 }
9094
9095 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
9096 // ABI says: "One of the simple integer types no more than 64 bits wide.
9097 // ... If such an argument is shorter than 64 bits, replace it by a full
9098 // 64-bit integer representing the same number, using sign or zero
9099 // extension". Shadow for an integer argument has the same type as the
9100 // argument itself, so it can be sign or zero extended as well.
9101 bool ZExt = CB.paramHasAttr(ArgNo, Kind: Attribute::ZExt);
9102 bool SExt = CB.paramHasAttr(ArgNo, Kind: Attribute::SExt);
9103 if (ZExt) {
9104 assert(!SExt);
9105 return ShadowExtension::Zero;
9106 }
9107 if (SExt) {
9108 assert(!ZExt);
9109 return ShadowExtension::Sign;
9110 }
9111 return ShadowExtension::None;
9112 }
9113
9114 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9115 unsigned GpOffset = SystemZGpOffset;
9116 unsigned FpOffset = SystemZFpOffset;
9117 unsigned VrIndex = 0;
9118 unsigned OverflowOffset = SystemZOverflowOffset;
9119 const DataLayout &DL = F.getDataLayout();
9120 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9121 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9122 // SystemZABIInfo does not produce ByVal parameters.
9123 assert(!CB.isByValArgument(ArgNo));
9124 Type *T = A->getType();
9125 ArgKind AK = classifyArgument(T);
9126 if (AK == ArgKind::Indirect) {
9127 T = MS.PtrTy;
9128 AK = ArgKind::GeneralPurpose;
9129 }
9130 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
9131 AK = ArgKind::Memory;
9132 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
9133 AK = ArgKind::Memory;
9134 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
9135 AK = ArgKind::Memory;
9136 Value *ShadowBase = nullptr;
9137 Value *OriginBase = nullptr;
9138 ShadowExtension SE = ShadowExtension::None;
9139 switch (AK) {
9140 case ArgKind::GeneralPurpose: {
9141 // Always keep track of GpOffset, but store shadow only for varargs.
9142 uint64_t ArgSize = 8;
9143 if (GpOffset + ArgSize <= kParamTLSSize) {
9144 if (!IsFixed) {
9145 SE = getShadowExtension(CB, ArgNo);
9146 uint64_t GapSize = 0;
9147 if (SE == ShadowExtension::None) {
9148 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9149 assert(ArgAllocSize <= ArgSize);
9150 GapSize = ArgSize - ArgAllocSize;
9151 }
9152 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9153 if (MS.TrackOrigins)
9154 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9155 }
9156 GpOffset += ArgSize;
9157 } else {
9158 GpOffset = kParamTLSSize;
9159 }
9160 break;
9161 }
9162 case ArgKind::FloatingPoint: {
9163 // Always keep track of FpOffset, but store shadow only for varargs.
9164 uint64_t ArgSize = 8;
9165 if (FpOffset + ArgSize <= kParamTLSSize) {
9166 if (!IsFixed) {
9167 // PoP says: "A short floating-point datum requires only the
9168 // left-most 32 bit positions of a floating-point register".
9169 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
9170 // don't extend shadow and don't mind the gap.
9171 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: FpOffset);
9172 if (MS.TrackOrigins)
9173 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
9174 }
9175 FpOffset += ArgSize;
9176 } else {
9177 FpOffset = kParamTLSSize;
9178 }
9179 break;
9180 }
9181 case ArgKind::Vector: {
9182 // Keep track of VrIndex. No need to store shadow, since vector varargs
9183 // go through AK_Memory.
9184 assert(IsFixed);
9185 VrIndex++;
9186 break;
9187 }
9188 case ArgKind::Memory: {
9189 // Keep track of OverflowOffset and store shadow only for varargs.
9190 // Ignore fixed args, since we need to copy only the vararg portion of
9191 // the overflow area shadow.
9192 if (!IsFixed) {
9193 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9194 uint64_t ArgSize = alignTo(Value: ArgAllocSize, Align: 8);
9195 if (OverflowOffset + ArgSize <= kParamTLSSize) {
9196 SE = getShadowExtension(CB, ArgNo);
9197 uint64_t GapSize =
9198 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
9199 ShadowBase =
9200 getShadowAddrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9201 if (MS.TrackOrigins)
9202 OriginBase =
9203 getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9204 OverflowOffset += ArgSize;
9205 } else {
9206 OverflowOffset = kParamTLSSize;
9207 }
9208 }
9209 break;
9210 }
9211 case ArgKind::Indirect:
9212 llvm_unreachable("Indirect must be converted to GeneralPurpose");
9213 }
9214 if (ShadowBase == nullptr)
9215 continue;
9216 Value *Shadow = MSV.getShadow(V: A);
9217 if (SE != ShadowExtension::None)
9218 Shadow = MSV.CreateShadowCast(IRB, V: Shadow, dstTy: IRB.getInt64Ty(),
9219 /*Signed*/ SE == ShadowExtension::Sign);
9220 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: MS.PtrTy, Name: "_msarg_va_s");
9221 IRB.CreateStore(Val: Shadow, Ptr: ShadowBase);
9222 if (MS.TrackOrigins) {
9223 Value *Origin = MSV.getOrigin(V: A);
9224 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
9225 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
9226 Alignment: kMinOriginAlignment);
9227 }
9228 }
9229 Constant *OverflowSize = ConstantInt::get(
9230 Ty: IRB.getInt64Ty(), V: OverflowOffset - SystemZOverflowOffset);
9231 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
9232 }
9233
9234 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
9235 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
9236 V: IRB.CreateAdd(
9237 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9238 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZRegSaveAreaPtrOffset)),
9239 DestTy: MS.PtrTy);
9240 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9241 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9242 const Align Alignment = Align(8);
9243 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9244 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
9245 /*isStore*/ true);
9246 // TODO(iii): copy only fragments filled by visitCallBase()
9247 // TODO(iii): support packed-stack && !use-soft-float
9248 // For use-soft-float functions, it is enough to copy just the GPRs.
9249 unsigned RegSaveAreaSize =
9250 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
9251 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9252 Size: RegSaveAreaSize);
9253 if (MS.TrackOrigins)
9254 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
9255 SrcAlign: Alignment, Size: RegSaveAreaSize);
9256 }
9257
9258 // FIXME: This implementation limits OverflowOffset to kParamTLSSize, so we
9259 // don't know real overflow size and can't clear shadow beyond kParamTLSSize.
9260 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
9261 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
9262 V: IRB.CreateAdd(
9263 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9264 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowArgAreaPtrOffset)),
9265 DestTy: MS.PtrTy);
9266 Value *OverflowArgAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
9267 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
9268 const Align Alignment = Align(8);
9269 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
9270 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9271 Alignment, /*isStore*/ true);
9272 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
9273 Idx0: SystemZOverflowOffset);
9274 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9275 Size: VAArgOverflowSize);
9276 if (MS.TrackOrigins) {
9277 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
9278 Idx0: SystemZOverflowOffset);
9279 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9280 Size: VAArgOverflowSize);
9281 }
9282 }
9283
9284 void finalizeInstrumentation() override {
9285 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
9286 "finalizeInstrumentation called twice");
9287 if (!VAStartInstrumentationList.empty()) {
9288 // If there is a va_start in this function, make a backup copy of
9289 // va_arg_tls somewhere in the function entry block.
9290 IRBuilder<> IRB(MSV.FnPrologueEnd);
9291 VAArgOverflowSize =
9292 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
9293 Value *CopySize =
9294 IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowOffset),
9295 RHS: VAArgOverflowSize);
9296 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9297 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9298 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9299 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9300
9301 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9302 ID: Intrinsic::umin, LHS: CopySize,
9303 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9304 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9305 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9306 if (MS.TrackOrigins) {
9307 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9308 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
9309 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
9310 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9311 }
9312 }
9313
9314 // Instrument va_start.
9315 // Copy va_list shadow from the backup copy of the TLS contents.
9316 for (CallInst *OrigInst : VAStartInstrumentationList) {
9317 NextNodeIRBuilder IRB(OrigInst);
9318 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9319 copyRegSaveArea(IRB, VAListTag);
9320 copyOverflowArea(IRB, VAListTag);
9321 }
9322 }
9323};
9324
9325/// i386-specific implementation of VarArgHelper.
9326struct VarArgI386Helper : public VarArgHelperBase {
9327 AllocaInst *VAArgTLSCopy = nullptr;
9328 Value *VAArgSize = nullptr;
9329
9330 VarArgI386Helper(Function &F, MemorySanitizer &MS,
9331 MemorySanitizerVisitor &MSV)
9332 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/4) {}
9333
9334 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9335 const DataLayout &DL = F.getDataLayout();
9336 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9337 unsigned VAArgOffset = 0;
9338 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9339 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9340 bool IsByVal = CB.isByValArgument(ArgNo);
9341 if (IsByVal) {
9342 assert(A->getType()->isPointerTy());
9343 Type *RealTy = CB.getParamByValType(ArgNo);
9344 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
9345 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
9346 if (ArgAlign < IntptrSize)
9347 ArgAlign = Align(IntptrSize);
9348 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9349 if (!IsFixed) {
9350 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9351 if (Base) {
9352 Value *AShadowPtr, *AOriginPtr;
9353 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
9354 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
9355 Alignment: kShadowTLSAlignment, /*isStore*/ false);
9356
9357 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
9358 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
9359 }
9360 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9361 }
9362 } else {
9363 Value *Base;
9364 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9365 Align ArgAlign = Align(IntptrSize);
9366 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9367 if (DL.isBigEndian()) {
9368 // Adjusting the shadow for argument with size < IntptrSize to match
9369 // the placement of bits in big endian system
9370 if (ArgSize < IntptrSize)
9371 VAArgOffset += (IntptrSize - ArgSize);
9372 }
9373 if (!IsFixed) {
9374 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9375 if (Base)
9376 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9377 VAArgOffset += ArgSize;
9378 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9379 }
9380 }
9381 }
9382
9383 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9384 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9385 // a new class member i.e. it is the total size of all VarArgs.
9386 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9387 }
9388
9389 void finalizeInstrumentation() override {
9390 assert(!VAArgSize && !VAArgTLSCopy &&
9391 "finalizeInstrumentation called twice");
9392 IRBuilder<> IRB(MSV.FnPrologueEnd);
9393 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9394 Value *CopySize = VAArgSize;
9395
9396 if (!VAStartInstrumentationList.empty()) {
9397 // If there is a va_start in this function, make a backup copy of
9398 // va_arg_tls somewhere in the function entry block.
9399 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9400 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9401 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9402 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9403
9404 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9405 ID: Intrinsic::umin, LHS: CopySize,
9406 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9407 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9408 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9409 }
9410
9411 // Instrument va_start.
9412 // Copy va_list shadow from the backup copy of the TLS contents.
9413 for (CallInst *OrigInst : VAStartInstrumentationList) {
9414 NextNodeIRBuilder IRB(OrigInst);
9415 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9416 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9417 Value *RegSaveAreaPtrPtr =
9418 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9419 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9420 Value *RegSaveAreaPtr =
9421 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9422 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9423 const DataLayout &DL = F.getDataLayout();
9424 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9425 const Align Alignment = Align(IntptrSize);
9426 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9427 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9428 Alignment, /*isStore*/ true);
9429 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9430 Size: CopySize);
9431 }
9432 }
9433};
9434
9435/// Implementation of VarArgHelper that is used for ARM32, MIPS, RISCV,
9436/// LoongArch64.
9437struct VarArgGenericHelper : public VarArgHelperBase {
9438 AllocaInst *VAArgTLSCopy = nullptr;
9439 Value *VAArgSize = nullptr;
9440
9441 VarArgGenericHelper(Function &F, MemorySanitizer &MS,
9442 MemorySanitizerVisitor &MSV, const unsigned VAListTagSize)
9443 : VarArgHelperBase(F, MS, MSV, VAListTagSize) {}
9444
9445 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9446 unsigned VAArgOffset = 0;
9447 const DataLayout &DL = F.getDataLayout();
9448 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9449 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9450 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9451 if (IsFixed)
9452 continue;
9453 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9454 if (DL.isBigEndian()) {
9455 // Adjusting the shadow for argument with size < IntptrSize to match the
9456 // placement of bits in big endian system
9457 if (ArgSize < IntptrSize)
9458 VAArgOffset += (IntptrSize - ArgSize);
9459 }
9460 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9461 VAArgOffset += ArgSize;
9462 VAArgOffset = alignTo(Value: VAArgOffset, Align: IntptrSize);
9463 if (!Base)
9464 continue;
9465 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9466 }
9467
9468 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9469 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9470 // a new class member i.e. it is the total size of all VarArgs.
9471 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9472 }
9473
9474 void finalizeInstrumentation() override {
9475 assert(!VAArgSize && !VAArgTLSCopy &&
9476 "finalizeInstrumentation called twice");
9477 IRBuilder<> IRB(MSV.FnPrologueEnd);
9478 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9479 Value *CopySize = VAArgSize;
9480
9481 if (!VAStartInstrumentationList.empty()) {
9482 // If there is a va_start in this function, make a backup copy of
9483 // va_arg_tls somewhere in the function entry block.
9484 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9485 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9486 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9487 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9488
9489 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9490 ID: Intrinsic::umin, LHS: CopySize,
9491 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9492 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9493 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9494 }
9495
9496 // Instrument va_start.
9497 // Copy va_list shadow from the backup copy of the TLS contents.
9498 for (CallInst *OrigInst : VAStartInstrumentationList) {
9499 NextNodeIRBuilder IRB(OrigInst);
9500 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9501 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9502 Value *RegSaveAreaPtrPtr =
9503 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9504 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9505 Value *RegSaveAreaPtr =
9506 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9507 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9508 const DataLayout &DL = F.getDataLayout();
9509 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9510 const Align Alignment = Align(IntptrSize);
9511 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9512 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9513 Alignment, /*isStore*/ true);
9514 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9515 Size: CopySize);
9516 }
9517 }
9518};
9519
9520// ARM32, Loongarch64, MIPS and RISCV share the same calling conventions
9521// regarding VAArgs.
9522using VarArgARM32Helper = VarArgGenericHelper;
9523using VarArgRISCVHelper = VarArgGenericHelper;
9524using VarArgMIPSHelper = VarArgGenericHelper;
9525using VarArgLoongArch64Helper = VarArgGenericHelper;
9526using VarArgHexagonHelper = VarArgGenericHelper;
9527
9528/// A no-op implementation of VarArgHelper.
9529struct VarArgNoOpHelper : public VarArgHelper {
9530 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
9531 MemorySanitizerVisitor &MSV) {}
9532
9533 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
9534
9535 void visitVAStartInst(VAStartInst &I) override {}
9536
9537 void visitVACopyInst(VACopyInst &I) override {}
9538
9539 void finalizeInstrumentation() override {}
9540};
9541
9542} // end anonymous namespace
9543
9544static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
9545 MemorySanitizerVisitor &Visitor) {
9546 // VarArg handling is only implemented on AMD64. False positives are possible
9547 // on other platforms.
9548 Triple TargetTriple(Func.getParent()->getTargetTriple());
9549
9550 if (TargetTriple.getArch() == Triple::x86)
9551 return new VarArgI386Helper(Func, Msan, Visitor);
9552
9553 if (TargetTriple.getArch() == Triple::x86_64)
9554 return new VarArgAMD64Helper(Func, Msan, Visitor);
9555
9556 if (TargetTriple.isARM())
9557 return new VarArgARM32Helper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9558
9559 if (TargetTriple.isAArch64())
9560 return new VarArgAArch64Helper(Func, Msan, Visitor);
9561
9562 if (TargetTriple.isSystemZ())
9563 return new VarArgSystemZHelper(Func, Msan, Visitor);
9564
9565 // On PowerPC32 VAListTag is a struct
9566 // {char, char, i16 padding, char *, char *}
9567 if (TargetTriple.isPPC32())
9568 return new VarArgPowerPC32Helper(Func, Msan, Visitor);
9569
9570 if (TargetTriple.isPPC64())
9571 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
9572
9573 if (TargetTriple.isRISCV32())
9574 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9575
9576 if (TargetTriple.isRISCV64())
9577 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9578
9579 if (TargetTriple.isMIPS32())
9580 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9581
9582 if (TargetTriple.isMIPS64())
9583 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9584
9585 if (TargetTriple.isLoongArch64())
9586 return new VarArgLoongArch64Helper(Func, Msan, Visitor,
9587 /*VAListTagSize=*/8);
9588
9589 if (TargetTriple.getArch() == Triple::hexagon)
9590 return new VarArgHexagonHelper(Func, Msan, Visitor, /*VAListTagSize=*/12);
9591
9592 return new VarArgNoOpHelper(Func, Msan, Visitor);
9593}
9594
9595bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
9596 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
9597 return false;
9598
9599 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
9600 return false;
9601
9602 MemorySanitizerVisitor Visitor(F, *this, TLI);
9603
9604 // Clear out memory attributes.
9605 AttributeMask B;
9606 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
9607 F.removeFnAttrs(Attrs: B);
9608
9609 return Visitor.runOnFunction();
9610}
9611