1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html)
15/// We associate a few shadow bits with every byte of the application memory,
16/// poison the shadow of the malloc-ed or alloca-ed memory, load the shadow,
17/// bits on every memory read, propagate the shadow bits through some of the
18/// arithmetic instruction (including MOV), store the shadow bits on every
19/// memory write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// Note that the sanitizer code has to deal with how shadow/origin pairs
126/// returned by the these functions are represented in different ABIs. In
127/// the X86_64 ABI they are returned in RDX:RAX, in PowerPC64 they are
128/// returned in r3 and r4, and in the SystemZ ABI they are written to memory
129/// pointed to by a hidden parameter.
130/// - TLS variables are stored in a single per-task struct. A call to a
131/// function __msan_get_context_state() returning a pointer to that struct
132/// is inserted into every instrumented function before the entry block;
133/// - __msan_warning() takes a 32-bit origin parameter;
134/// - local variables are poisoned with __msan_poison_alloca() upon function
135/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
136/// function;
137/// - the pass doesn't declare any global variables or add global constructors
138/// to the translation unit.
139///
140/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
141/// calls, making sure we're on the safe side wrt. possible false positives.
142///
143/// KernelMemorySanitizer only supports X86_64, SystemZ and PowerPC64 at the
144/// moment.
145///
146//
147// FIXME: This sanitizer does not yet handle scalable vectors
148//
149//===----------------------------------------------------------------------===//
150
151#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
152#include "llvm/ADT/APInt.h"
153#include "llvm/ADT/ArrayRef.h"
154#include "llvm/ADT/DenseMap.h"
155#include "llvm/ADT/DepthFirstIterator.h"
156#include "llvm/ADT/SetVector.h"
157#include "llvm/ADT/SmallPtrSet.h"
158#include "llvm/ADT/SmallVector.h"
159#include "llvm/ADT/StringExtras.h"
160#include "llvm/ADT/StringRef.h"
161#include "llvm/Analysis/GlobalsModRef.h"
162#include "llvm/Analysis/TargetLibraryInfo.h"
163#include "llvm/Analysis/ValueTracking.h"
164#include "llvm/IR/Argument.h"
165#include "llvm/IR/AttributeMask.h"
166#include "llvm/IR/Attributes.h"
167#include "llvm/IR/BasicBlock.h"
168#include "llvm/IR/CallingConv.h"
169#include "llvm/IR/Constant.h"
170#include "llvm/IR/Constants.h"
171#include "llvm/IR/DataLayout.h"
172#include "llvm/IR/DerivedTypes.h"
173#include "llvm/IR/Function.h"
174#include "llvm/IR/GlobalValue.h"
175#include "llvm/IR/GlobalVariable.h"
176#include "llvm/IR/IRBuilder.h"
177#include "llvm/IR/InlineAsm.h"
178#include "llvm/IR/InstVisitor.h"
179#include "llvm/IR/InstrTypes.h"
180#include "llvm/IR/Instruction.h"
181#include "llvm/IR/Instructions.h"
182#include "llvm/IR/IntrinsicInst.h"
183#include "llvm/IR/Intrinsics.h"
184#include "llvm/IR/IntrinsicsAArch64.h"
185#include "llvm/IR/IntrinsicsX86.h"
186#include "llvm/IR/MDBuilder.h"
187#include "llvm/IR/Module.h"
188#include "llvm/IR/Type.h"
189#include "llvm/IR/Value.h"
190#include "llvm/IR/ValueMap.h"
191#include "llvm/Support/Alignment.h"
192#include "llvm/Support/AtomicOrdering.h"
193#include "llvm/Support/Casting.h"
194#include "llvm/Support/CommandLine.h"
195#include "llvm/Support/Debug.h"
196#include "llvm/Support/DebugCounter.h"
197#include "llvm/Support/ErrorHandling.h"
198#include "llvm/Support/MathExtras.h"
199#include "llvm/Support/raw_ostream.h"
200#include "llvm/TargetParser/Triple.h"
201#include "llvm/Transforms/Utils/BasicBlockUtils.h"
202#include "llvm/Transforms/Utils/Instrumentation.h"
203#include "llvm/Transforms/Utils/Local.h"
204#include "llvm/Transforms/Utils/ModuleUtils.h"
205#include <algorithm>
206#include <cassert>
207#include <cstddef>
208#include <cstdint>
209#include <memory>
210#include <numeric>
211#include <string>
212#include <tuple>
213
214using namespace llvm;
215
216#define DEBUG_TYPE "msan"
217
218DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
219 "Controls which checks to insert");
220
221DEBUG_COUNTER(DebugInstrumentInstruction, "msan-instrument-instruction",
222 "Controls which instruction to instrument");
223
224static const unsigned kOriginSize = 4;
225static const Align kMinOriginAlignment = Align(4);
226static const Align kShadowTLSAlignment = Align(8);
227
228// These constants must be kept in sync with the ones in msan.h.
229// TODO: increase size to match SVE/SVE2/SME/SME2 limits
230static const unsigned kParamTLSSize = 800;
231static const unsigned kRetvalTLSSize = 800;
232
233// Accesses sizes are powers of two: 1, 2, 4, 8.
234static const size_t kNumberOfAccessSizes = 4;
235
236/// Track origins of uninitialized values.
237///
238/// Adds a section to MemorySanitizer report that points to the allocation
239/// (stack or heap) the uninitialized bits came from originally.
240static cl::opt<int> ClTrackOrigins(
241 "msan-track-origins",
242 cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden,
243 cl::init(Val: 0));
244
245static cl::opt<bool> ClKeepGoing("msan-keep-going",
246 cl::desc("keep going after reporting a UMR"),
247 cl::Hidden, cl::init(Val: false));
248
249static cl::opt<bool>
250 ClPoisonStack("msan-poison-stack",
251 cl::desc("poison uninitialized stack variables"), cl::Hidden,
252 cl::init(Val: true));
253
254static cl::opt<bool> ClPoisonStackWithCall(
255 "msan-poison-stack-with-call",
256 cl::desc("poison uninitialized stack variables with a call"), cl::Hidden,
257 cl::init(Val: false));
258
259static cl::opt<int> ClPoisonStackPattern(
260 "msan-poison-stack-pattern",
261 cl::desc("poison uninitialized stack variables with the given pattern"),
262 cl::Hidden, cl::init(Val: 0xff));
263
264static cl::opt<bool>
265 ClPrintStackNames("msan-print-stack-names",
266 cl::desc("Print name of local stack variable"),
267 cl::Hidden, cl::init(Val: true));
268
269static cl::opt<bool>
270 ClPoisonUndef("msan-poison-undef",
271 cl::desc("Poison fully undef temporary values. "
272 "Partially undefined constant vectors "
273 "are unaffected by this flag (see "
274 "-msan-poison-undef-vectors)."),
275 cl::Hidden, cl::init(Val: true));
276
277static cl::opt<bool> ClPoisonUndefVectors(
278 "msan-poison-undef-vectors",
279 cl::desc("Precisely poison partially undefined constant vectors. "
280 "If false (legacy behavior), the entire vector is "
281 "considered fully initialized, which may lead to false "
282 "negatives. Fully undefined constant vectors are "
283 "unaffected by this flag (see -msan-poison-undef)."),
284 cl::Hidden, cl::init(Val: false));
285
286static cl::opt<bool> ClPreciseDisjointOr(
287 "msan-precise-disjoint-or",
288 cl::desc("Precisely poison disjoint OR. If false (legacy behavior), "
289 "disjointedness is ignored (i.e., 1|1 is initialized)."),
290 cl::Hidden, cl::init(Val: false));
291
292static cl::opt<bool>
293 ClHandleICmp("msan-handle-icmp",
294 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
295 cl::Hidden, cl::init(Val: true));
296
297static cl::opt<bool>
298 ClHandleICmpExact("msan-handle-icmp-exact",
299 cl::desc("exact handling of relational integer ICmp"),
300 cl::Hidden, cl::init(Val: true));
301
302static cl::opt<int> ClSwitchPrecision(
303 "msan-switch-precision",
304 cl::desc("Controls the number of cases considered by MSan for LLVM switch "
305 "instructions. 0 means no UUMs detected. Higher values lead to "
306 "fewer false negatives but may impact compiler and/or "
307 "application performance. N.B. LLVM switch instructions do not "
308 "correspond exactly to C++ switch statements."),
309 cl::Hidden, cl::init(Val: 99));
310
311static cl::opt<bool> ClHandleLifetimeIntrinsics(
312 "msan-handle-lifetime-intrinsics",
313 cl::desc(
314 "when possible, poison scoped variables at the beginning of the scope "
315 "(slower, but more precise)"),
316 cl::Hidden, cl::init(Val: true));
317
318// When compiling the Linux kernel, we sometimes see false positives related to
319// MSan being unable to understand that inline assembly calls may initialize
320// local variables.
321// This flag makes the compiler conservatively unpoison every memory location
322// passed into an assembly call. Note that this may cause false positives.
323// Because it's impossible to figure out the array sizes, we can only unpoison
324// the first sizeof(type) bytes for each type* pointer.
325static cl::opt<bool> ClHandleAsmConservative(
326 "msan-handle-asm-conservative",
327 cl::desc("conservative handling of inline assembly"), cl::Hidden,
328 cl::init(Val: true));
329
330// This flag controls whether we check the shadow of the address
331// operand of load or store. Such bugs are very rare, since load from
332// a garbage address typically results in SEGV, but still happen
333// (e.g. only lower bits of address are garbage, or the access happens
334// early at program startup where malloc-ed memory is more likely to
335// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
336static cl::opt<bool> ClCheckAccessAddress(
337 "msan-check-access-address",
338 cl::desc("report accesses through a pointer which has poisoned shadow"),
339 cl::Hidden, cl::init(Val: true));
340
341static cl::opt<bool> ClEagerChecks(
342 "msan-eager-checks",
343 cl::desc("check arguments and return values at function call boundaries"),
344 cl::Hidden, cl::init(Val: false));
345
346static cl::opt<bool> ClDumpStrictInstructions(
347 "msan-dump-strict-instructions",
348 cl::desc("print out instructions with default strict semantics i.e.,"
349 "check that all the inputs are fully initialized, and mark "
350 "the output as fully initialized. These semantics are applied "
351 "to instructions that could not be handled explicitly nor "
352 "heuristically."),
353 cl::Hidden, cl::init(Val: false));
354
355// Currently, all the heuristically handled instructions are specifically
356// IntrinsicInst. However, we use the broader "HeuristicInstructions" name
357// to parallel 'msan-dump-strict-instructions', and to keep the door open to
358// handling non-intrinsic instructions heuristically.
359static cl::opt<bool> ClDumpHeuristicInstructions(
360 "msan-dump-heuristic-instructions",
361 cl::desc("Prints 'unknown' instructions that were handled heuristically. "
362 "Use -msan-dump-strict-instructions to print instructions that "
363 "could not be handled explicitly nor heuristically."),
364 cl::Hidden, cl::init(Val: false));
365
366static cl::opt<int> ClInstrumentationWithCallThreshold(
367 "msan-instrumentation-with-call-threshold",
368 cl::desc(
369 "If the function being instrumented requires more than "
370 "this number of checks and origin stores, use callbacks instead of "
371 "inline checks (-1 means never use callbacks)."),
372 cl::Hidden, cl::init(Val: 3500));
373
374static cl::opt<bool>
375 ClEnableKmsan("msan-kernel",
376 cl::desc("Enable KernelMemorySanitizer instrumentation"),
377 cl::Hidden, cl::init(Val: false));
378
379static cl::opt<bool>
380 ClDisableChecks("msan-disable-checks",
381 cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
382 cl::init(Val: false));
383
384static cl::opt<bool>
385 ClCheckConstantShadow("msan-check-constant-shadow",
386 cl::desc("Insert checks for constant shadow values"),
387 cl::Hidden, cl::init(Val: true));
388
389// This is off by default because of a bug in gold:
390// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
391static cl::opt<bool>
392 ClWithComdat("msan-with-comdat",
393 cl::desc("Place MSan constructors in comdat sections"),
394 cl::Hidden, cl::init(Val: false));
395
396// These options allow to specify custom memory map parameters
397// See MemoryMapParams for details.
398static cl::opt<uint64_t> ClAndMask("msan-and-mask",
399 cl::desc("Define custom MSan AndMask"),
400 cl::Hidden, cl::init(Val: 0));
401
402static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
403 cl::desc("Define custom MSan XorMask"),
404 cl::Hidden, cl::init(Val: 0));
405
406static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
407 cl::desc("Define custom MSan ShadowBase"),
408 cl::Hidden, cl::init(Val: 0));
409
410static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
411 cl::desc("Define custom MSan OriginBase"),
412 cl::Hidden, cl::init(Val: 0));
413
414static cl::opt<int>
415 ClDisambiguateWarning("msan-disambiguate-warning-threshold",
416 cl::desc("Define threshold for number of checks per "
417 "debug location to force origin update."),
418 cl::Hidden, cl::init(Val: 3));
419
420const char kMsanModuleCtorName[] = "msan.module_ctor";
421const char kMsanInitName[] = "__msan_init";
422
423namespace {
424
425// Memory map parameters used in application-to-shadow address calculation.
426// Offset = (Addr & ~AndMask) ^ XorMask
427// Shadow = ShadowBase + Offset
428// Origin = OriginBase + Offset
429struct MemoryMapParams {
430 uint64_t AndMask;
431 uint64_t XorMask;
432 uint64_t ShadowBase;
433 uint64_t OriginBase;
434};
435
436struct PlatformMemoryMapParams {
437 const MemoryMapParams *bits32;
438 const MemoryMapParams *bits64;
439};
440
441} // end anonymous namespace
442
443// i386 Linux
444static const MemoryMapParams Linux_I386_MemoryMapParams = {
445 .AndMask: 0x000080000000, // AndMask
446 .XorMask: 0, // XorMask (not used)
447 .ShadowBase: 0, // ShadowBase (not used)
448 .OriginBase: 0x000040000000, // OriginBase
449};
450
451// x86_64 Linux
452static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
453 .AndMask: 0, // AndMask (not used)
454 .XorMask: 0x500000000000, // XorMask
455 .ShadowBase: 0, // ShadowBase (not used)
456 .OriginBase: 0x100000000000, // OriginBase
457};
458
459// mips32 Linux
460// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
461// after picking good constants
462
463// mips64 Linux
464static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
465 .AndMask: 0, // AndMask (not used)
466 .XorMask: 0x008000000000, // XorMask
467 .ShadowBase: 0, // ShadowBase (not used)
468 .OriginBase: 0x002000000000, // OriginBase
469};
470
471// ppc32 Linux
472// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
473// after picking good constants
474
475// ppc64 Linux
476static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
477 .AndMask: 0xE00000000000, // AndMask
478 .XorMask: 0x100000000000, // XorMask
479 .ShadowBase: 0x080000000000, // ShadowBase
480 .OriginBase: 0x1C0000000000, // OriginBase
481};
482
483// s390x Linux
484static const MemoryMapParams Linux_S390X_MemoryMapParams = {
485 .AndMask: 0xC00000000000, // AndMask
486 .XorMask: 0, // XorMask (not used)
487 .ShadowBase: 0x080000000000, // ShadowBase
488 .OriginBase: 0x1C0000000000, // OriginBase
489};
490
491// arm32 Linux
492// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
493// after picking good constants
494
495// aarch64 Linux
496static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
497 .AndMask: 0, // AndMask (not used)
498 .XorMask: 0x0B00000000000, // XorMask
499 .ShadowBase: 0, // ShadowBase (not used)
500 .OriginBase: 0x0200000000000, // OriginBase
501};
502
503// loongarch64 Linux
504static const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
505 .AndMask: 0, // AndMask (not used)
506 .XorMask: 0x500000000000, // XorMask
507 .ShadowBase: 0, // ShadowBase (not used)
508 .OriginBase: 0x100000000000, // OriginBase
509};
510
511// hexagon Linux
512static const MemoryMapParams Linux_Hexagon_MemoryMapParams = {
513 .AndMask: 0, // AndMask (not used)
514 .XorMask: 0x20000000, // XorMask
515 .ShadowBase: 0, // ShadowBase (not used)
516 .OriginBase: 0x50000000, // OriginBase
517};
518
519// riscv32 Linux
520// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
521// after picking good constants
522
523// aarch64 FreeBSD
524static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
525 .AndMask: 0x1800000000000, // AndMask
526 .XorMask: 0x0400000000000, // XorMask
527 .ShadowBase: 0x0200000000000, // ShadowBase
528 .OriginBase: 0x0700000000000, // OriginBase
529};
530
531// i386 FreeBSD
532static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
533 .AndMask: 0x000180000000, // AndMask
534 .XorMask: 0x000040000000, // XorMask
535 .ShadowBase: 0x000020000000, // ShadowBase
536 .OriginBase: 0x000700000000, // OriginBase
537};
538
539// x86_64 FreeBSD
540static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
541 .AndMask: 0xc00000000000, // AndMask
542 .XorMask: 0x200000000000, // XorMask
543 .ShadowBase: 0x100000000000, // ShadowBase
544 .OriginBase: 0x380000000000, // OriginBase
545};
546
547// x86_64 NetBSD
548static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
549 .AndMask: 0, // AndMask
550 .XorMask: 0x500000000000, // XorMask
551 .ShadowBase: 0, // ShadowBase
552 .OriginBase: 0x100000000000, // OriginBase
553};
554
555static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
556 .bits32: &Linux_I386_MemoryMapParams,
557 .bits64: &Linux_X86_64_MemoryMapParams,
558};
559
560static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
561 .bits32: nullptr,
562 .bits64: &Linux_MIPS64_MemoryMapParams,
563};
564
565static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
566 .bits32: nullptr,
567 .bits64: &Linux_PowerPC64_MemoryMapParams,
568};
569
570static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
571 .bits32: nullptr,
572 .bits64: &Linux_S390X_MemoryMapParams,
573};
574
575static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
576 .bits32: nullptr,
577 .bits64: &Linux_AArch64_MemoryMapParams,
578};
579
580static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams = {
581 .bits32: nullptr,
582 .bits64: &Linux_LoongArch64_MemoryMapParams,
583};
584
585static const PlatformMemoryMapParams Linux_Hexagon_MemoryMapParams_P = {
586 .bits32: &Linux_Hexagon_MemoryMapParams,
587 .bits64: nullptr,
588};
589
590static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
591 .bits32: nullptr,
592 .bits64: &FreeBSD_AArch64_MemoryMapParams,
593};
594
595static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
596 .bits32: &FreeBSD_I386_MemoryMapParams,
597 .bits64: &FreeBSD_X86_64_MemoryMapParams,
598};
599
600static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
601 .bits32: nullptr,
602 .bits64: &NetBSD_X86_64_MemoryMapParams,
603};
604
605enum OddOrEvenLanes { kBothLanes, kEvenLanes, kOddLanes };
606
607namespace {
608
609/// Instrument functions of a module to detect uninitialized reads.
610///
611/// Instantiating MemorySanitizer inserts the msan runtime library API function
612/// declarations into the module if they don't exist already. Instantiating
613/// ensures the __msan_init function is in the list of global constructors for
614/// the module.
615class MemorySanitizer {
616public:
617 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
618 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
619 Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
620 initializeModule(M);
621 }
622
623 // MSan cannot be moved or copied because of MapParams.
624 MemorySanitizer(MemorySanitizer &&) = delete;
625 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
626 MemorySanitizer(const MemorySanitizer &) = delete;
627 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
628
629 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
630
631private:
632 friend struct MemorySanitizerVisitor;
633 friend struct VarArgHelperBase;
634 friend struct VarArgAMD64Helper;
635 friend struct VarArgAArch64Helper;
636 friend struct VarArgPowerPC64Helper;
637 friend struct VarArgPowerPC32Helper;
638 friend struct VarArgSystemZHelper;
639 friend struct VarArgI386Helper;
640 friend struct VarArgGenericHelper;
641
642 void initializeModule(Module &M);
643 void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
644 void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
645 void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
646
647 template <typename... ArgsTy>
648 FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
649 ArgsTy... Args);
650
651 /// True if we're compiling the Linux kernel.
652 bool CompileKernel;
653 /// Track origins (allocation points) of uninitialized values.
654 int TrackOrigins;
655 bool Recover;
656 bool EagerChecks;
657
658 Triple TargetTriple;
659 LLVMContext *C;
660 Type *IntptrTy; ///< Integer type with the size of a ptr in default AS.
661 Type *OriginTy;
662 PointerType *PtrTy; ///< Integer type with the size of a ptr in default AS.
663
664 // XxxTLS variables represent the per-thread state in MSan and per-task state
665 // in KMSAN.
666 // For the userspace these point to thread-local globals. In the kernel land
667 // they point to the members of a per-task struct obtained via a call to
668 // __msan_get_context_state().
669
670 /// Thread-local shadow storage for function parameters.
671 Value *ParamTLS;
672
673 /// Thread-local origin storage for function parameters.
674 Value *ParamOriginTLS;
675
676 /// Thread-local shadow storage for function return value.
677 Value *RetvalTLS;
678
679 /// Thread-local origin storage for function return value.
680 Value *RetvalOriginTLS;
681
682 /// Thread-local shadow storage for in-register va_arg function.
683 Value *VAArgTLS;
684
685 /// Thread-local shadow storage for in-register va_arg function.
686 Value *VAArgOriginTLS;
687
688 /// Thread-local shadow storage for va_arg overflow area.
689 Value *VAArgOverflowSizeTLS;
690
691 /// Are the instrumentation callbacks set up?
692 bool CallbacksInitialized = false;
693
694 /// The run-time callback to print a warning.
695 FunctionCallee WarningFn;
696
697 // These arrays are indexed by log2(AccessSize).
698 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
699 FunctionCallee MaybeWarningVarSizeFn;
700 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
701
702 /// Run-time helper that generates a new origin value for a stack
703 /// allocation.
704 FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
705 // No description version
706 FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
707
708 /// Run-time helper that poisons stack on function entry.
709 FunctionCallee MsanPoisonStackFn;
710
711 /// Run-time helper that records a store (or any event) of an
712 /// uninitialized value and returns an updated origin id encoding this info.
713 FunctionCallee MsanChainOriginFn;
714
715 /// Run-time helper that paints an origin over a region.
716 FunctionCallee MsanSetOriginFn;
717
718 /// MSan runtime replacements for memmove, memcpy and memset.
719 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
720
721 /// KMSAN callback for task-local function argument shadow.
722 StructType *MsanContextStateTy;
723 FunctionCallee MsanGetContextStateFn;
724
725 /// Functions for poisoning/unpoisoning local variables
726 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
727
728 /// Pair of shadow/origin pointers.
729 Type *MsanMetadata;
730
731 /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
732 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
733 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
734 FunctionCallee MsanMetadataPtrForStore_1_8[4];
735 FunctionCallee MsanInstrumentAsmStoreFn;
736
737 /// Storage for return values of the MsanMetadataPtrXxx functions.
738 Value *MsanMetadataAlloca;
739
740 /// Helper to choose between different MsanMetadataPtrXxx().
741 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
742
743 /// Memory map parameters used in application-to-shadow calculation.
744 const MemoryMapParams *MapParams;
745
746 /// Custom memory map parameters used when -msan-shadow-base or
747 // -msan-origin-base is provided.
748 MemoryMapParams CustomMapParams;
749
750 MDNode *ColdCallWeights;
751
752 /// Branch weights for origin store.
753 MDNode *OriginStoreWeights;
754};
755
756void insertModuleCtor(Module &M) {
757 getOrCreateSanitizerCtorAndInitFunctions(
758 M, CtorName: kMsanModuleCtorName, InitName: kMsanInitName,
759 /*InitArgTypes=*/{},
760 /*InitArgs=*/{},
761 // This callback is invoked when the functions are created the first
762 // time. Hook them into the global ctors list in that case:
763 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
764 if (!ClWithComdat) {
765 appendToGlobalCtors(M, F: Ctor, Priority: 0);
766 return;
767 }
768 Comdat *MsanCtorComdat = M.getOrInsertComdat(Name: kMsanModuleCtorName);
769 Ctor->setComdat(MsanCtorComdat);
770 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
771 });
772}
773
774template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
775 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
776}
777
778} // end anonymous namespace
779
780MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K,
781 bool EagerChecks)
782 : Kernel(getOptOrDefault(Opt: ClEnableKmsan, Default: K)),
783 TrackOrigins(getOptOrDefault(Opt: ClTrackOrigins, Default: Kernel ? 2 : TO)),
784 Recover(getOptOrDefault(Opt: ClKeepGoing, Default: Kernel || R)),
785 EagerChecks(getOptOrDefault(Opt: ClEagerChecks, Default: EagerChecks)) {}
786
787PreservedAnalyses MemorySanitizerPass::run(Module &M,
788 ModuleAnalysisManager &AM) {
789 // Return early if nosanitize_memory module flag is present for the module.
790 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_memory"))
791 return PreservedAnalyses::all();
792 bool Modified = false;
793 if (!Options.Kernel) {
794 insertModuleCtor(M);
795 Modified = true;
796 }
797
798 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
799 for (Function &F : M) {
800 if (F.empty())
801 continue;
802 MemorySanitizer Msan(*F.getParent(), Options);
803 Modified |=
804 Msan.sanitizeFunction(F, TLI&: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
805 }
806
807 if (!Modified)
808 return PreservedAnalyses::all();
809
810 PreservedAnalyses PA = PreservedAnalyses::none();
811 // GlobalsAA is considered stateless and does not get invalidated unless
812 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
813 // make changes that require GlobalsAA to be invalidated.
814 PA.abandon<GlobalsAA>();
815 return PA;
816}
817
818void MemorySanitizerPass::printPipeline(
819 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
820 static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
821 OS, MapClassName2PassName);
822 OS << '<';
823 if (Options.Recover)
824 OS << "recover;";
825 if (Options.Kernel)
826 OS << "kernel;";
827 if (Options.EagerChecks)
828 OS << "eager-checks;";
829 OS << "track-origins=" << Options.TrackOrigins;
830 OS << '>';
831}
832
833/// Create a non-const global initialized with the given string.
834///
835/// Creates a writable global for Str so that we can pass it to the
836/// run-time lib. Runtime uses first 4 bytes of the string to store the
837/// frame ID, so the string needs to be mutable.
838static GlobalVariable *createPrivateConstGlobalForString(Module &M,
839 StringRef Str) {
840 Constant *StrConst = ConstantDataArray::getString(Context&: M.getContext(), Initializer: Str);
841 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
842 GlobalValue::PrivateLinkage, StrConst, "");
843}
844
845template <typename... ArgsTy>
846FunctionCallee
847MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
848 ArgsTy... Args) {
849 if (TargetTriple.getArch() == Triple::systemz) {
850 // SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
851 return M.getOrInsertFunction(Name, Type::getVoidTy(C&: *C), PtrTy,
852 std::forward<ArgsTy>(Args)...);
853 }
854
855 return M.getOrInsertFunction(Name, MsanMetadata,
856 std::forward<ArgsTy>(Args)...);
857}
858
859/// Create KMSAN API callbacks.
860void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
861 IRBuilder<> IRB(*C);
862
863 // These will be initialized in insertKmsanPrologue().
864 RetvalTLS = nullptr;
865 RetvalOriginTLS = nullptr;
866 ParamTLS = nullptr;
867 ParamOriginTLS = nullptr;
868 VAArgTLS = nullptr;
869 VAArgOriginTLS = nullptr;
870 VAArgOverflowSizeTLS = nullptr;
871
872 WarningFn = M.getOrInsertFunction(Name: "__msan_warning",
873 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
874 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
875
876 // Requests the per-task context state (kmsan_context_state*) from the
877 // runtime library.
878 MsanContextStateTy = StructType::get(
879 elt1: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
880 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8),
881 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
882 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8), /* va_arg_origin */
883 elts: IRB.getInt64Ty(), elts: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4), elts: OriginTy,
884 elts: OriginTy);
885 MsanGetContextStateFn =
886 M.getOrInsertFunction(Name: "__msan_get_context_state", RetTy: PtrTy);
887
888 MsanMetadata = StructType::get(elt1: PtrTy, elts: PtrTy);
889
890 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
891 std::string name_load =
892 "__msan_metadata_ptr_for_load_" + std::to_string(val: size);
893 std::string name_store =
894 "__msan_metadata_ptr_for_store_" + std::to_string(val: size);
895 MsanMetadataPtrForLoad_1_8[ind] =
896 getOrInsertMsanMetadataFunction(M, Name: name_load, Args: PtrTy);
897 MsanMetadataPtrForStore_1_8[ind] =
898 getOrInsertMsanMetadataFunction(M, Name: name_store, Args: PtrTy);
899 }
900
901 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
902 M, Name: "__msan_metadata_ptr_for_load_n", Args: PtrTy, Args: IntptrTy);
903 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
904 M, Name: "__msan_metadata_ptr_for_store_n", Args: PtrTy, Args: IntptrTy);
905
906 // Functions for poisoning and unpoisoning memory.
907 MsanPoisonAllocaFn = M.getOrInsertFunction(
908 Name: "__msan_poison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
909 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
910 Name: "__msan_unpoison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
911}
912
913static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
914 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&] {
915 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
916 nullptr, Name, nullptr,
917 GlobalVariable::InitialExecTLSModel);
918 });
919}
920
921/// Insert declarations for userspace-specific functions and globals.
922void MemorySanitizer::createUserspaceApi(Module &M,
923 const TargetLibraryInfo &TLI) {
924 IRBuilder<> IRB(*C);
925
926 // Create the callback.
927 // FIXME: this function should have "Cold" calling conv,
928 // which is not yet implemented.
929 if (TrackOrigins) {
930 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
931 : "__msan_warning_with_origin_noreturn";
932 WarningFn = M.getOrInsertFunction(Name: WarningFnName,
933 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
934 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
935 } else {
936 StringRef WarningFnName =
937 Recover ? "__msan_warning" : "__msan_warning_noreturn";
938 WarningFn = M.getOrInsertFunction(Name: WarningFnName, RetTy: IRB.getVoidTy());
939 }
940
941 // Create the global TLS variables.
942 RetvalTLS =
943 getOrInsertGlobal(M, Name: "__msan_retval_tls",
944 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8));
945
946 RetvalOriginTLS = getOrInsertGlobal(M, Name: "__msan_retval_origin_tls", Ty: OriginTy);
947
948 ParamTLS =
949 getOrInsertGlobal(M, Name: "__msan_param_tls",
950 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
951
952 ParamOriginTLS =
953 getOrInsertGlobal(M, Name: "__msan_param_origin_tls",
954 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
955
956 VAArgTLS =
957 getOrInsertGlobal(M, Name: "__msan_va_arg_tls",
958 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
959
960 VAArgOriginTLS =
961 getOrInsertGlobal(M, Name: "__msan_va_arg_origin_tls",
962 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
963
964 VAArgOverflowSizeTLS = getOrInsertGlobal(M, Name: "__msan_va_arg_overflow_size_tls",
965 Ty: IRB.getIntPtrTy(DL: M.getDataLayout()));
966
967 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
968 AccessSizeIndex++) {
969 unsigned AccessSize = 1 << AccessSizeIndex;
970 std::string FunctionName = "__msan_maybe_warning_" + itostr(X: AccessSize);
971 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
972 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 1}, /*Signed=*/false),
973 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: IRB.getInt32Ty());
974 MaybeWarningVarSizeFn = M.getOrInsertFunction(
975 Name: "__msan_maybe_warning_N", AttributeList: TLI.getAttrList(C, ArgNos: {}, /*Signed=*/false),
976 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IRB.getInt64Ty(), Args: IRB.getInt32Ty());
977 FunctionName = "__msan_maybe_store_origin_" + itostr(X: AccessSize);
978 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
979 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 2}, /*Signed=*/false),
980 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: PtrTy,
981 Args: IRB.getInt32Ty());
982 }
983
984 MsanSetAllocaOriginWithDescriptionFn =
985 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_with_descr",
986 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy, Args: PtrTy);
987 MsanSetAllocaOriginNoDescriptionFn =
988 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_no_descr",
989 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
990 MsanPoisonStackFn = M.getOrInsertFunction(Name: "__msan_poison_stack",
991 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
992}
993
994/// Insert extern declaration of runtime-provided functions and globals.
995void MemorySanitizer::initializeCallbacks(Module &M,
996 const TargetLibraryInfo &TLI) {
997 // Only do this once.
998 if (CallbacksInitialized)
999 return;
1000
1001 IRBuilder<> IRB(*C);
1002 // Initialize callbacks that are common for kernel and userspace
1003 // instrumentation.
1004 MsanChainOriginFn = M.getOrInsertFunction(
1005 Name: "__msan_chain_origin",
1006 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false, /*Ret=*/true), RetTy: IRB.getInt32Ty(),
1007 Args: IRB.getInt32Ty());
1008 MsanSetOriginFn = M.getOrInsertFunction(
1009 Name: "__msan_set_origin", AttributeList: TLI.getAttrList(C, ArgNos: {2}, /*Signed=*/false),
1010 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: IRB.getInt32Ty());
1011 MemmoveFn =
1012 M.getOrInsertFunction(Name: "__msan_memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
1013 MemcpyFn =
1014 M.getOrInsertFunction(Name: "__msan_memcpy", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
1015 MemsetFn = M.getOrInsertFunction(Name: "__msan_memset",
1016 AttributeList: TLI.getAttrList(C, ArgNos: {1}, /*Signed=*/true),
1017 RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
1018
1019 MsanInstrumentAsmStoreFn = M.getOrInsertFunction(
1020 Name: "__msan_instrument_asm_store", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
1021
1022 if (CompileKernel) {
1023 createKernelApi(M, TLI);
1024 } else {
1025 createUserspaceApi(M, TLI);
1026 }
1027 CallbacksInitialized = true;
1028}
1029
1030FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
1031 int size) {
1032 FunctionCallee *Fns =
1033 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
1034 switch (size) {
1035 case 1:
1036 return Fns[0];
1037 case 2:
1038 return Fns[1];
1039 case 4:
1040 return Fns[2];
1041 case 8:
1042 return Fns[3];
1043 default:
1044 return nullptr;
1045 }
1046}
1047
1048/// Module-level initialization.
1049///
1050/// inserts a call to __msan_init to the module's constructor list.
1051void MemorySanitizer::initializeModule(Module &M) {
1052 auto &DL = M.getDataLayout();
1053
1054 TargetTriple = M.getTargetTriple();
1055
1056 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
1057 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
1058 // Check the overrides first
1059 if (ShadowPassed || OriginPassed) {
1060 CustomMapParams.AndMask = ClAndMask;
1061 CustomMapParams.XorMask = ClXorMask;
1062 CustomMapParams.ShadowBase = ClShadowBase;
1063 CustomMapParams.OriginBase = ClOriginBase;
1064 MapParams = &CustomMapParams;
1065 } else {
1066 switch (TargetTriple.getOS()) {
1067 case Triple::FreeBSD:
1068 switch (TargetTriple.getArch()) {
1069 case Triple::aarch64:
1070 MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
1071 break;
1072 case Triple::x86_64:
1073 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
1074 break;
1075 case Triple::x86:
1076 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
1077 break;
1078 default:
1079 report_fatal_error(reason: "unsupported architecture");
1080 }
1081 break;
1082 case Triple::NetBSD:
1083 switch (TargetTriple.getArch()) {
1084 case Triple::x86_64:
1085 MapParams = NetBSD_X86_MemoryMapParams.bits64;
1086 break;
1087 default:
1088 report_fatal_error(reason: "unsupported architecture");
1089 }
1090 break;
1091 case Triple::Linux:
1092 switch (TargetTriple.getArch()) {
1093 case Triple::x86_64:
1094 MapParams = Linux_X86_MemoryMapParams.bits64;
1095 break;
1096 case Triple::x86:
1097 MapParams = Linux_X86_MemoryMapParams.bits32;
1098 break;
1099 case Triple::mips64:
1100 case Triple::mips64el:
1101 MapParams = Linux_MIPS_MemoryMapParams.bits64;
1102 break;
1103 case Triple::ppc64:
1104 case Triple::ppc64le:
1105 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
1106 break;
1107 case Triple::systemz:
1108 MapParams = Linux_S390_MemoryMapParams.bits64;
1109 break;
1110 case Triple::aarch64:
1111 case Triple::aarch64_be:
1112 MapParams = Linux_ARM_MemoryMapParams.bits64;
1113 break;
1114 case Triple::loongarch64:
1115 MapParams = Linux_LoongArch_MemoryMapParams.bits64;
1116 break;
1117 case Triple::hexagon:
1118 MapParams = Linux_Hexagon_MemoryMapParams_P.bits32;
1119 break;
1120 default:
1121 report_fatal_error(reason: "unsupported architecture");
1122 }
1123 break;
1124 default:
1125 report_fatal_error(reason: "unsupported operating system");
1126 }
1127 }
1128
1129 C = &(M.getContext());
1130 IRBuilder<> IRB(*C);
1131 IntptrTy = IRB.getIntPtrTy(DL);
1132 OriginTy = IRB.getInt32Ty();
1133 PtrTy = IRB.getPtrTy();
1134
1135 ColdCallWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1136 OriginStoreWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1137
1138 if (!CompileKernel) {
1139 if (TrackOrigins)
1140 M.getOrInsertGlobal(Name: "__msan_track_origins", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1141 return new GlobalVariable(
1142 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1143 IRB.getInt32(C: TrackOrigins), "__msan_track_origins");
1144 });
1145
1146 if (Recover)
1147 M.getOrInsertGlobal(Name: "__msan_keep_going", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1148 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1149 GlobalValue::WeakODRLinkage,
1150 IRB.getInt32(C: Recover), "__msan_keep_going");
1151 });
1152 }
1153}
1154
1155namespace {
1156
1157/// A helper class that handles instrumentation of VarArg
1158/// functions on a particular platform.
1159///
1160/// Implementations are expected to insert the instrumentation
1161/// necessary to propagate argument shadow through VarArg function
1162/// calls. Visit* methods are called during an InstVisitor pass over
1163/// the function, and should avoid creating new basic blocks. A new
1164/// instance of this class is created for each instrumented function.
1165struct VarArgHelper {
1166 virtual ~VarArgHelper() = default;
1167
1168 /// Visit a CallBase.
1169 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1170
1171 /// Visit a va_start call.
1172 virtual void visitVAStartInst(VAStartInst &I) = 0;
1173
1174 /// Visit a va_copy call.
1175 virtual void visitVACopyInst(VACopyInst &I) = 0;
1176
1177 /// Finalize function instrumentation.
1178 ///
1179 /// This method is called after visiting all interesting (see above)
1180 /// instructions in a function.
1181 virtual void finalizeInstrumentation() = 0;
1182};
1183
1184struct MemorySanitizerVisitor;
1185
1186} // end anonymous namespace
1187
1188static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1189 MemorySanitizerVisitor &Visitor);
1190
1191static unsigned TypeSizeToSizeIndex(TypeSize TS) {
1192 if (TS.isScalable())
1193 // Scalable types unconditionally take slowpaths.
1194 return kNumberOfAccessSizes;
1195 unsigned TypeSizeFixed = TS.getFixedValue();
1196 if (TypeSizeFixed <= 8)
1197 return 0;
1198 return Log2_32_Ceil(Value: (TypeSizeFixed + 7) / 8);
1199}
1200
1201namespace {
1202
1203/// Helper class to attach debug information of the given instruction onto new
1204/// instructions inserted after.
1205class NextNodeIRBuilder : public IRBuilder<> {
1206public:
1207 explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1208 SetCurrentDebugLocation(IP->getDebugLoc());
1209 }
1210};
1211
1212/// This class does all the work for a given function. Store and Load
1213/// instructions store and load corresponding shadow and origin
1214/// values. Most instructions propagate shadow from arguments to their
1215/// return values. Certain instructions (most importantly, BranchInst)
1216/// test their argument shadow and print reports (with a runtime call) if it's
1217/// non-zero.
1218struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1219 Function &F;
1220 MemorySanitizer &MS;
1221 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1222 ValueMap<Value *, Value *> ShadowMap, OriginMap;
1223 std::unique_ptr<VarArgHelper> VAHelper;
1224 const TargetLibraryInfo *TLI;
1225 Instruction *FnPrologueEnd;
1226 SmallVector<Instruction *, 16> Instructions;
1227
1228 // The following flags disable parts of MSan instrumentation based on
1229 // exclusion list contents and command-line options.
1230 bool InsertChecks;
1231 bool PropagateShadow;
1232 bool PoisonStack;
1233 bool PoisonUndef;
1234 bool PoisonUndefVectors;
1235
1236 struct ShadowOriginAndInsertPoint {
1237 Value *Shadow;
1238 Value *Origin;
1239 Instruction *OrigIns;
1240
1241 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1242 : Shadow(S), Origin(O), OrigIns(I) {}
1243 };
1244 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1245 DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
1246 SmallSetVector<AllocaInst *, 16> AllocaSet;
1247 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1248 SmallVector<StoreInst *, 16> StoreList;
1249 int64_t SplittableBlocksCount = 0;
1250
1251 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1252 const TargetLibraryInfo &TLI)
1253 : F(F), MS(MS), VAHelper(CreateVarArgHelper(Func&: F, Msan&: MS, Visitor&: *this)), TLI(&TLI) {
1254 bool SanitizeFunction =
1255 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) && !ClDisableChecks;
1256 InsertChecks = SanitizeFunction;
1257 PropagateShadow = SanitizeFunction;
1258 PoisonStack = SanitizeFunction && ClPoisonStack;
1259 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1260 PoisonUndefVectors = SanitizeFunction && ClPoisonUndefVectors;
1261
1262 // In the presence of unreachable blocks, we may see Phi nodes with
1263 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1264 // blocks, such nodes will not have any shadow value associated with them.
1265 // It's easier to remove unreachable blocks than deal with missing shadow.
1266 removeUnreachableBlocks(F);
1267
1268 MS.initializeCallbacks(M&: *F.getParent(), TLI);
1269 FnPrologueEnd =
1270 IRBuilder<>(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt())
1271 .CreateIntrinsicWithoutFolding(ID: Intrinsic::donothing, Args: {});
1272
1273 if (MS.CompileKernel) {
1274 IRBuilder<> IRB(FnPrologueEnd);
1275 insertKmsanPrologue(IRB);
1276 }
1277
1278 LLVM_DEBUG(if (!InsertChecks) dbgs()
1279 << "MemorySanitizer is not inserting checks into '"
1280 << F.getName() << "'\n");
1281 }
1282
1283 bool instrumentWithCalls(Value *V) {
1284 // Constants likely will be eliminated by follow-up passes.
1285 if (isa<Constant>(Val: V))
1286 return false;
1287 ++SplittableBlocksCount;
1288 return ClInstrumentationWithCallThreshold >= 0 &&
1289 SplittableBlocksCount > ClInstrumentationWithCallThreshold;
1290 }
1291
1292 bool isInPrologue(Instruction &I) {
1293 return I.getParent() == FnPrologueEnd->getParent() &&
1294 (&I == FnPrologueEnd || I.comesBefore(Other: FnPrologueEnd));
1295 }
1296
1297 // Creates a new origin and records the stack trace. In general we can call
1298 // this function for any origin manipulation we like. However it will cost
1299 // runtime resources. So use this wisely only if it can provide additional
1300 // information helpful to a user.
1301 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1302 if (MS.TrackOrigins <= 1)
1303 return V;
1304 return IRB.CreateCall(Callee: MS.MsanChainOriginFn, Args: V);
1305 }
1306
1307 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1308 const DataLayout &DL = F.getDataLayout();
1309 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1310 if (IntptrSize == kOriginSize)
1311 return Origin;
1312 assert(IntptrSize == kOriginSize * 2);
1313 Origin = IRB.CreateIntCast(V: Origin, DestTy: MS.IntptrTy, /* isSigned */ false);
1314 return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: kOriginSize * 8));
1315 }
1316
1317 /// Fill memory range with the given origin value.
1318 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1319 TypeSize TS, Align Alignment) {
1320 const DataLayout &DL = F.getDataLayout();
1321 const Align IntptrAlignment = DL.getABITypeAlign(Ty: MS.IntptrTy);
1322 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1323 assert(IntptrAlignment >= kMinOriginAlignment);
1324 assert(IntptrSize >= kOriginSize);
1325
1326 // Note: The loop based formation works for fixed length vectors too,
1327 // however we prefer to unroll and specialize alignment below.
1328 if (TS.isScalable()) {
1329 Value *Size = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size: TS);
1330 Value *RoundUp =
1331 IRB.CreateAdd(LHS: Size, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize - 1));
1332 Value *End =
1333 IRB.CreateUDiv(LHS: RoundUp, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize));
1334 auto [InsertPt, Index] =
1335 SplitBlockAndInsertSimpleForLoop(End, SplitBefore: IRB.GetInsertPoint());
1336 IRB.SetInsertPoint(InsertPt);
1337
1338 Value *GEP = IRB.CreateGEP(Ty: MS.OriginTy, Ptr: OriginPtr, IdxList: Index);
1339 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: kMinOriginAlignment);
1340 return;
1341 }
1342
1343 unsigned Size = TS.getFixedValue();
1344
1345 unsigned Ofs = 0;
1346 Align CurrentAlignment = Alignment;
1347 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1348 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1349 Value *IntptrOriginPtr = IRB.CreatePointerCast(V: OriginPtr, DestTy: MS.PtrTy);
1350 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1351 Value *Ptr = i ? IRB.CreateConstGEP1_32(Ty: MS.IntptrTy, Ptr: IntptrOriginPtr, Idx0: i)
1352 : IntptrOriginPtr;
1353 IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
1354 Ofs += IntptrSize / kOriginSize;
1355 CurrentAlignment = IntptrAlignment;
1356 }
1357 }
1358
1359 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1360 Value *GEP =
1361 i ? IRB.CreateConstGEP1_32(Ty: MS.OriginTy, Ptr: OriginPtr, Idx0: i) : OriginPtr;
1362 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
1363 CurrentAlignment = kMinOriginAlignment;
1364 }
1365 }
1366
1367 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1368 Value *OriginPtr, Align Alignment) {
1369 const DataLayout &DL = F.getDataLayout();
1370 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1371 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
1372 // ZExt cannot convert between vector and scalar
1373 Value *ConvertedShadow = convertShadowToScalar(V: Shadow, IRB);
1374 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1375 if (!ClCheckConstantShadow || ConstantShadow->isNullValue()) {
1376 // Origin is not needed: value is initialized or const shadow is
1377 // ignored.
1378 return;
1379 }
1380 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1381 // Copy origin as the value is definitely uninitialized.
1382 paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), OriginPtr, TS: StoreSize,
1383 Alignment: OriginAlignment);
1384 return;
1385 }
1386 // Fallback to runtime check, which still can be optimized out later.
1387 }
1388
1389 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1390 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1391 if (instrumentWithCalls(V: ConvertedShadow) &&
1392 SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1393 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1394 Value *ConvertedShadow2 =
1395 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1396 CallBase *CB = IRB.CreateCall(Callee: Fn, Args: {ConvertedShadow2, Addr, Origin});
1397 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1398 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1399 } else {
1400 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1401 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1402 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: MS.OriginStoreWeights);
1403 IRBuilder<> IRBNew(CheckTerm);
1404 paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), OriginPtr, TS: StoreSize,
1405 Alignment: OriginAlignment);
1406 }
1407 }
1408
1409 void materializeStores() {
1410 for (StoreInst *SI : StoreList) {
1411 IRBuilder<> IRB(SI);
1412 Value *Val = SI->getValueOperand();
1413 Value *Addr = SI->getPointerOperand();
1414 Value *Shadow = SI->isAtomic() ? getCleanShadow(V: Val) : getShadow(V: Val);
1415 Value *ShadowPtr, *OriginPtr;
1416 Type *ShadowTy = Shadow->getType();
1417 const Align Alignment = SI->getAlign();
1418 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1419 std::tie(args&: ShadowPtr, args&: OriginPtr) =
1420 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1421
1422 [[maybe_unused]] StoreInst *NewSI =
1423 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Alignment);
1424 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1425
1426 if (SI->isAtomic())
1427 SI->setOrdering(addReleaseOrdering(a: SI->getOrdering()));
1428
1429 if (MS.TrackOrigins && !SI->isAtomic())
1430 storeOrigin(IRB, Addr, Shadow, Origin: getOrigin(V: Val), OriginPtr,
1431 Alignment: OriginAlignment);
1432 }
1433 }
1434
1435 // Returns true if Debug Location corresponds to multiple warnings.
1436 bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1437 if (MS.TrackOrigins < 2)
1438 return false;
1439
1440 if (LazyWarningDebugLocationCount.empty())
1441 for (const auto &I : InstrumentationList)
1442 ++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1443
1444 return LazyWarningDebugLocationCount[DebugLoc] >= ClDisambiguateWarning;
1445 }
1446
1447 /// Helper function to insert a warning at IRB's current insert point.
1448 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1449 if (!Origin)
1450 Origin = (Value *)IRB.getInt32(C: 0);
1451 assert(Origin->getType()->isIntegerTy());
1452
1453 if (shouldDisambiguateWarningLocation(DebugLoc: IRB.getCurrentDebugLocation())) {
1454 // Try to create additional origin with debug info of the last origin
1455 // instruction. It may provide additional information to the user.
1456 if (Instruction *OI = dyn_cast_or_null<Instruction>(Val: Origin)) {
1457 assert(MS.TrackOrigins);
1458 auto NewDebugLoc = OI->getDebugLoc();
1459 // Origin update with missing or the same debug location provides no
1460 // additional value.
1461 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1462 // Insert update just before the check, so we call runtime only just
1463 // before the report.
1464 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1465 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1466 Origin = updateOrigin(V: Origin, IRB&: IRBOrigin);
1467 }
1468 }
1469 }
1470
1471 if (MS.CompileKernel || MS.TrackOrigins)
1472 IRB.CreateCall(Callee: MS.WarningFn, Args: Origin)->setCannotMerge();
1473 else
1474 IRB.CreateCall(Callee: MS.WarningFn)->setCannotMerge();
1475 // FIXME: Insert UnreachableInst if !MS.Recover?
1476 // This may invalidate some of the following checks and needs to be done
1477 // at the very end.
1478 }
1479
1480 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1481 Value *Origin) {
1482 const DataLayout &DL = F.getDataLayout();
1483 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1484 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1485 if (instrumentWithCalls(V: ConvertedShadow) && !MS.CompileKernel) {
1486 // ZExt cannot convert between vector and scalar
1487 ConvertedShadow = convertShadowToScalar(V: ConvertedShadow, IRB);
1488 Value *ConvertedShadow2 =
1489 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1490
1491 if (SizeIndex < kNumberOfAccessSizes) {
1492 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1493 CallBase *CB = IRB.CreateCall(
1494 Callee: Fn,
1495 Args: {ConvertedShadow2,
1496 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1497 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1498 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1499 } else {
1500 FunctionCallee Fn = MS.MaybeWarningVarSizeFn;
1501 Value *ShadowAlloca = IRB.CreateAlloca(Ty: ConvertedShadow2->getType(), AddrSpace: 0u);
1502 IRB.CreateStore(Val: ConvertedShadow2, Ptr: ShadowAlloca);
1503 unsigned ShadowSize = DL.getTypeAllocSize(Ty: ConvertedShadow2->getType());
1504 CallBase *CB = IRB.CreateCall(
1505 Callee: Fn,
1506 Args: {ShadowAlloca, ConstantInt::get(Ty: IRB.getInt64Ty(), V: ShadowSize),
1507 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1508 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1509 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1510 }
1511 } else {
1512 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1513 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1514 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(),
1515 /* Unreachable */ !MS.Recover, BranchWeights: MS.ColdCallWeights);
1516
1517 IRB.SetInsertPoint(CheckTerm);
1518 insertWarningFn(IRB, Origin);
1519 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1520 }
1521 }
1522
1523 void materializeInstructionChecks(
1524 ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1525 const DataLayout &DL = F.getDataLayout();
1526 // Disable combining in some cases. TrackOrigins checks each shadow to pick
1527 // correct origin.
1528 bool Combine = !MS.TrackOrigins;
1529 Instruction *Instruction = InstructionChecks.front().OrigIns;
1530 Value *Shadow = nullptr;
1531 for (const auto &ShadowData : InstructionChecks) {
1532 assert(ShadowData.OrigIns == Instruction);
1533 IRBuilder<> IRB(Instruction);
1534
1535 Value *ConvertedShadow = ShadowData.Shadow;
1536
1537 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1538 if (!ClCheckConstantShadow || ConstantShadow->isNullValue()) {
1539 // Skip, value is initialized or const shadow is ignored.
1540 continue;
1541 }
1542 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1543 // Report as the value is definitely uninitialized.
1544 insertWarningFn(IRB, Origin: ShadowData.Origin);
1545 if (!MS.Recover)
1546 return; // Always fail and stop here, not need to check the rest.
1547 // Skip entire instruction,
1548 continue;
1549 }
1550 // Fallback to runtime check, which still can be optimized out later.
1551 }
1552
1553 if (!Combine) {
1554 materializeOneCheck(IRB, ConvertedShadow, Origin: ShadowData.Origin);
1555 continue;
1556 }
1557
1558 if (!Shadow) {
1559 Shadow = ConvertedShadow;
1560 continue;
1561 }
1562
1563 Shadow = convertToBool(V: Shadow, IRB, name: "_mscmp");
1564 ConvertedShadow = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1565 Shadow = IRB.CreateOr(LHS: Shadow, RHS: ConvertedShadow, Name: "_msor");
1566 }
1567
1568 if (Shadow) {
1569 assert(Combine);
1570 IRBuilder<> IRB(Instruction);
1571 materializeOneCheck(IRB, ConvertedShadow: Shadow, Origin: nullptr);
1572 }
1573 }
1574
1575 static bool isAArch64SVCount(Type *Ty) {
1576 if (TargetExtType *TTy = dyn_cast<TargetExtType>(Val: Ty))
1577 return TTy->getName() == "aarch64.svcount";
1578 return false;
1579 }
1580
1581 // This is intended to match the "AArch64 Predicate-as-Counter Type" (aka
1582 // 'target("aarch64.svcount")', but not e.g., <vscale x 4 x i32>.
1583 static bool isScalableNonVectorType(Type *Ty) {
1584 if (!isAArch64SVCount(Ty))
1585 LLVM_DEBUG(dbgs() << "isScalableNonVectorType: Unexpected type " << *Ty
1586 << "\n");
1587
1588 return Ty->isScalableTy() && !isa<VectorType>(Val: Ty);
1589 }
1590
1591 void materializeChecks() {
1592#ifndef NDEBUG
1593 // For assert below.
1594 SmallPtrSet<Instruction *, 16> Done;
1595#endif
1596
1597 for (auto I = InstrumentationList.begin();
1598 I != InstrumentationList.end();) {
1599 auto OrigIns = I->OrigIns;
1600 // Checks are grouped by the original instruction. We call all
1601 // `insertShadowCheck` for an instruction at once.
1602 assert(Done.insert(OrigIns).second);
1603 auto J = std::find_if(first: I + 1, last: InstrumentationList.end(),
1604 pred: [OrigIns](const ShadowOriginAndInsertPoint &R) {
1605 return OrigIns != R.OrigIns;
1606 });
1607 // Process all checks of instruction at once.
1608 materializeInstructionChecks(InstructionChecks: ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1609 I = J;
1610 }
1611
1612 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1613 }
1614
1615 // Returns the last instruction in the new prologue
1616 void insertKmsanPrologue(IRBuilder<> &IRB) {
1617 Value *ContextState = IRB.CreateCall(Callee: MS.MsanGetContextStateFn, Args: {});
1618 Constant *Zero = IRB.getInt32(C: 0);
1619 MS.ParamTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1620 IdxList: {Zero, IRB.getInt32(C: 0)}, Name: "param_shadow");
1621 MS.RetvalTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1622 IdxList: {Zero, IRB.getInt32(C: 1)}, Name: "retval_shadow");
1623 MS.VAArgTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1624 IdxList: {Zero, IRB.getInt32(C: 2)}, Name: "va_arg_shadow");
1625 MS.VAArgOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1626 IdxList: {Zero, IRB.getInt32(C: 3)}, Name: "va_arg_origin");
1627 MS.VAArgOverflowSizeTLS =
1628 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1629 IdxList: {Zero, IRB.getInt32(C: 4)}, Name: "va_arg_overflow_size");
1630 MS.ParamOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1631 IdxList: {Zero, IRB.getInt32(C: 5)}, Name: "param_origin");
1632 MS.RetvalOriginTLS =
1633 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1634 IdxList: {Zero, IRB.getInt32(C: 6)}, Name: "retval_origin");
1635 if (MS.TargetTriple.getArch() == Triple::systemz)
1636 MS.MsanMetadataAlloca = IRB.CreateAlloca(Ty: MS.MsanMetadata, AddrSpace: 0u);
1637 }
1638
1639 /// Add MemorySanitizer instrumentation to a function.
1640 bool runOnFunction() {
1641 // Iterate all BBs in depth-first order and create shadow instructions
1642 // for all instructions (where applicable).
1643 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1644 for (BasicBlock *BB : depth_first(G: FnPrologueEnd->getParent()))
1645 visit(BB&: *BB);
1646
1647 // `visit` above only collects instructions. Process them after iterating
1648 // CFG to avoid requirement on CFG transformations.
1649 for (Instruction *I : Instructions)
1650 InstVisitor<MemorySanitizerVisitor>::visit(I&: *I);
1651
1652 // Finalize PHI nodes.
1653 for (PHINode *PN : ShadowPHINodes) {
1654 PHINode *PNS = cast<PHINode>(Val: getShadow(V: PN));
1655 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(Val: getOrigin(V: PN)) : nullptr;
1656 size_t NumValues = PN->getNumIncomingValues();
1657 for (size_t v = 0; v < NumValues; v++) {
1658 PNS->addIncoming(V: getShadow(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1659 if (PNO)
1660 PNO->addIncoming(V: getOrigin(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1661 }
1662 }
1663
1664 VAHelper->finalizeInstrumentation();
1665
1666 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1667 // instrumenting only allocas.
1668 if (ClHandleLifetimeIntrinsics) {
1669 for (auto Item : LifetimeStartList) {
1670 instrumentAlloca(I&: *Item.second, InsPoint: Item.first);
1671 AllocaSet.remove(X: Item.second);
1672 }
1673 }
1674 // Poison the allocas for which we didn't instrument the corresponding
1675 // lifetime intrinsics.
1676 for (AllocaInst *AI : AllocaSet)
1677 instrumentAlloca(I&: *AI);
1678
1679 // Insert shadow value checks.
1680 materializeChecks();
1681
1682 // Delayed instrumentation of StoreInst.
1683 // This may not add new address checks.
1684 materializeStores();
1685
1686 return true;
1687 }
1688
1689 /// Compute the shadow type that corresponds to a given Value.
1690 Type *getShadowTy(Value *V) { return getShadowTy(OrigTy: V->getType()); }
1691
1692 /// Compute the shadow type that corresponds to a given Type.
1693 Type *getShadowTy(Type *OrigTy) {
1694 if (!OrigTy->isSized()) {
1695 return nullptr;
1696 }
1697 // For integer type, shadow is the same as the original type.
1698 // This may return weird-sized types like i1.
1699 if (IntegerType *IT = dyn_cast<IntegerType>(Val: OrigTy))
1700 return IT;
1701 const DataLayout &DL = F.getDataLayout();
1702 if (VectorType *VT = dyn_cast<VectorType>(Val: OrigTy)) {
1703 uint32_t EltSize = DL.getTypeSizeInBits(Ty: VT->getElementType());
1704 return VectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSize),
1705 EC: VT->getElementCount());
1706 }
1707 if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy)) {
1708 return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1709 NumElements: AT->getNumElements());
1710 }
1711 if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1712 SmallVector<Type *, 4> Elements;
1713 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1714 Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: i)));
1715 StructType *Res = StructType::get(Context&: *MS.C, Elements, isPacked: ST->isPacked());
1716 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1717 return Res;
1718 }
1719 if (isScalableNonVectorType(Ty: OrigTy)) {
1720 LLVM_DEBUG(dbgs() << "getShadowTy: Scalable non-vector type: " << *OrigTy
1721 << "\n");
1722 return OrigTy;
1723 }
1724
1725 uint32_t TypeSize = DL.getTypeSizeInBits(Ty: OrigTy);
1726 return IntegerType::get(C&: *MS.C, NumBits: TypeSize);
1727 }
1728
1729 /// Extract combined shadow of struct elements as a bool
1730 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1731 IRBuilder<> &IRB) {
1732 Value *FalseVal = IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1733 Value *Aggregator = FalseVal;
1734
1735 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1736 // Combine by ORing together each element's bool shadow
1737 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1738 Value *ShadowBool = convertToBool(V: ShadowItem, IRB);
1739
1740 if (Aggregator != FalseVal)
1741 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowBool);
1742 else
1743 Aggregator = ShadowBool;
1744 }
1745
1746 return Aggregator;
1747 }
1748
1749 // Extract combined shadow of array elements
1750 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1751 IRBuilder<> &IRB) {
1752 if (!Array->getNumElements())
1753 return IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1754
1755 Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
1756 Value *Aggregator = convertShadowToScalar(V: FirstItem, IRB);
1757
1758 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1759 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1760 Value *ShadowInner = convertShadowToScalar(V: ShadowItem, IRB);
1761 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
1762 }
1763 return Aggregator;
1764 }
1765
1766 /// Convert a shadow value to it's flattened variant. The resulting
1767 /// shadow may not necessarily have the same bit width as the input
1768 /// value, but it will always be comparable to zero.
1769 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1770 if (StructType *Struct = dyn_cast<StructType>(Val: V->getType()))
1771 return collapseStructShadow(Struct, Shadow: V, IRB);
1772 if (ArrayType *Array = dyn_cast<ArrayType>(Val: V->getType()))
1773 return collapseArrayShadow(Array, Shadow: V, IRB);
1774 if (isa<VectorType>(Val: V->getType())) {
1775 if (isa<ScalableVectorType>(Val: V->getType()))
1776 return convertShadowToScalar(V: IRB.CreateOrReduce(Src: V), IRB);
1777 unsigned BitWidth =
1778 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1779 return IRB.CreateBitCast(V, DestTy: IntegerType::get(C&: *MS.C, NumBits: BitWidth));
1780 }
1781 return V;
1782 }
1783
1784 // Convert a scalar value to an i1 by comparing with 0
1785 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1786 Type *VTy = V->getType();
1787 if (!VTy->isIntegerTy())
1788 return convertToBool(V: convertShadowToScalar(V, IRB), IRB, name);
1789 if (VTy->getIntegerBitWidth() == 1)
1790 // Just converting a bool to a bool, so do nothing.
1791 return V;
1792 return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name: name);
1793 }
1794
1795 Type *ptrToIntPtrType(Type *PtrTy) const {
1796 if (VectorType *VectTy = dyn_cast<VectorType>(Val: PtrTy)) {
1797 return VectorType::get(ElementType: ptrToIntPtrType(PtrTy: VectTy->getElementType()),
1798 EC: VectTy->getElementCount());
1799 }
1800 assert(PtrTy->isIntOrPtrTy());
1801 return MS.IntptrTy;
1802 }
1803
1804 Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1805 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1806 return VectorType::get(
1807 ElementType: getPtrToShadowPtrType(IntPtrTy: VectTy->getElementType(), ShadowTy),
1808 EC: VectTy->getElementCount());
1809 }
1810 assert(IntPtrTy == MS.IntptrTy);
1811 return MS.PtrTy;
1812 }
1813
1814 Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1815 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1816 return ConstantVector::getSplat(
1817 EC: VectTy->getElementCount(),
1818 Elt: constToIntPtr(IntPtrTy: VectTy->getElementType(), C));
1819 }
1820 assert(IntPtrTy == MS.IntptrTy);
1821 // TODO: Avoid implicit trunc?
1822 // See https://github.com/llvm/llvm-project/issues/112510.
1823 return ConstantInt::get(Ty: MS.IntptrTy, V: C, /*IsSigned=*/false,
1824 /*ImplicitTrunc=*/true);
1825 }
1826
1827 /// Returns the integer shadow offset that corresponds to a given
1828 /// application address, whereby:
1829 ///
1830 /// Offset = (Addr & ~AndMask) ^ XorMask
1831 /// Shadow = ShadowBase + Offset
1832 /// Origin = (OriginBase + Offset) & ~Alignment
1833 ///
1834 /// Note: for efficiency, many shadow mappings only require use the XorMask
1835 /// and OriginBase; the AndMask and ShadowBase are often zero.
1836 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1837 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1838 Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1839
1840 if (uint64_t AndMask = MS.MapParams->AndMask)
1841 OffsetLong = IRB.CreateAnd(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~AndMask));
1842
1843 if (uint64_t XorMask = MS.MapParams->XorMask)
1844 OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: XorMask));
1845 return OffsetLong;
1846 }
1847
1848 /// Compute the shadow and origin addresses corresponding to a given
1849 /// application address.
1850 ///
1851 /// Shadow = ShadowBase + Offset
1852 /// Origin = (OriginBase + Offset) & ~3ULL
1853 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1854 /// a single pointee.
1855 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1856 std::pair<Value *, Value *>
1857 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1858 MaybeAlign Alignment) {
1859 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1860 if (!VectTy) {
1861 assert(Addr->getType()->isPointerTy());
1862 } else {
1863 assert(VectTy->getElementType()->isPointerTy());
1864 }
1865 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1866 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1867 Value *ShadowLong = ShadowOffset;
1868 if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1869 ShadowLong =
1870 IRB.CreateAdd(LHS: ShadowLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ShadowBase));
1871 }
1872 Value *ShadowPtr = IRB.CreateIntToPtr(
1873 V: ShadowLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy));
1874
1875 Value *OriginPtr = nullptr;
1876 if (MS.TrackOrigins) {
1877 Value *OriginLong = ShadowOffset;
1878 uint64_t OriginBase = MS.MapParams->OriginBase;
1879 if (OriginBase != 0)
1880 OriginLong =
1881 IRB.CreateAdd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: OriginBase));
1882 if (!Alignment || *Alignment < kMinOriginAlignment) {
1883 uint64_t Mask = kMinOriginAlignment.value() - 1;
1884 OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~Mask));
1885 }
1886 OriginPtr = IRB.CreateIntToPtr(
1887 V: OriginLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy: MS.OriginTy));
1888 }
1889 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1890 }
1891
1892 template <typename... ArgsTy>
1893 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1894 ArgsTy... Args) {
1895 if (MS.TargetTriple.getArch() == Triple::systemz) {
1896 IRB.CreateCall(Callee,
1897 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1898 return IRB.CreateLoad(Ty: MS.MsanMetadata, Ptr: MS.MsanMetadataAlloca);
1899 }
1900
1901 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1902 }
1903
1904 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
1905 IRBuilder<> &IRB,
1906 Type *ShadowTy,
1907 bool isStore) {
1908 Value *ShadowOriginPtrs;
1909 const DataLayout &DL = F.getDataLayout();
1910 TypeSize Size = DL.getTypeStoreSize(Ty: ShadowTy);
1911
1912 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, size: Size);
1913 Value *AddrCast = IRB.CreatePointerCast(V: Addr, DestTy: MS.PtrTy);
1914 if (Getter) {
1915 ShadowOriginPtrs = createMetadataCall(IRB, Callee: Getter, Args: AddrCast);
1916 } else {
1917 Value *SizeVal = ConstantInt::get(Ty: MS.IntptrTy, V: Size);
1918 ShadowOriginPtrs = createMetadataCall(
1919 IRB,
1920 Callee: isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1921 Args: AddrCast, Args: SizeVal);
1922 }
1923 Value *ShadowPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 0);
1924 ShadowPtr = IRB.CreatePointerCast(V: ShadowPtr, DestTy: MS.PtrTy);
1925 Value *OriginPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 1);
1926
1927 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1928 }
1929
1930 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1931 /// a single pointee.
1932 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1933 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1934 IRBuilder<> &IRB,
1935 Type *ShadowTy,
1936 bool isStore) {
1937 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1938 if (!VectTy) {
1939 assert(Addr->getType()->isPointerTy());
1940 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1941 }
1942
1943 // TODO: Support callbacs with vectors of addresses.
1944 unsigned NumElements = cast<FixedVectorType>(Val: VectTy)->getNumElements();
1945 Value *ShadowPtrs = ConstantInt::getNullValue(
1946 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1947 Value *OriginPtrs = nullptr;
1948 if (MS.TrackOrigins)
1949 OriginPtrs = ConstantInt::getNullValue(
1950 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1951 for (unsigned i = 0; i < NumElements; ++i) {
1952 Value *OneAddr =
1953 IRB.CreateExtractElement(Vec: Addr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1954 auto [ShadowPtr, OriginPtr] =
1955 getShadowOriginPtrKernelNoVec(Addr: OneAddr, IRB, ShadowTy, isStore);
1956
1957 ShadowPtrs = IRB.CreateInsertElement(
1958 Vec: ShadowPtrs, NewElt: ShadowPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1959 if (MS.TrackOrigins)
1960 OriginPtrs = IRB.CreateInsertElement(
1961 Vec: OriginPtrs, NewElt: OriginPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1962 }
1963 return {ShadowPtrs, OriginPtrs};
1964 }
1965
1966 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1967 Type *ShadowTy,
1968 MaybeAlign Alignment,
1969 bool isStore) {
1970 if (MS.CompileKernel)
1971 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1972 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1973 }
1974
1975 /// Compute the shadow address for a given function argument.
1976 ///
1977 /// Shadow = ParamTLS+ArgOffset.
1978 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1979 return IRB.CreatePtrAdd(Ptr: MS.ParamTLS,
1980 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg");
1981 }
1982
1983 /// Compute the origin address for a given function argument.
1984 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1985 if (!MS.TrackOrigins)
1986 return nullptr;
1987 return IRB.CreatePtrAdd(Ptr: MS.ParamOriginTLS,
1988 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
1989 Name: "_msarg_o");
1990 }
1991
1992 /// Compute the shadow address for a retval.
1993 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1994 return IRB.CreatePointerCast(V: MS.RetvalTLS, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msret");
1995 }
1996
1997 /// Compute the origin address for a retval.
1998 Value *getOriginPtrForRetval() {
1999 // We keep a single origin for the entire retval. Might be too optimistic.
2000 return MS.RetvalOriginTLS;
2001 }
2002
2003 /// Set SV to be the shadow value for V.
2004 void setShadow(Value *V, Value *SV) {
2005 assert(!ShadowMap.count(V) && "Values may only have one shadow");
2006 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
2007 }
2008
2009 /// Set Origin to be the origin value for V.
2010 void setOrigin(Value *V, Value *Origin) {
2011 if (!MS.TrackOrigins)
2012 return;
2013 assert(!OriginMap.count(V) && "Values may only have one origin");
2014 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
2015 OriginMap[V] = Origin;
2016 }
2017
2018 Constant *getCleanShadow(Type *OrigTy) {
2019 Type *ShadowTy = getShadowTy(OrigTy);
2020 if (!ShadowTy)
2021 return nullptr;
2022 return Constant::getNullValue(Ty: ShadowTy);
2023 }
2024
2025 /// Create a clean shadow value for a given value.
2026 ///
2027 /// Clean shadow (all zeroes) means all bits of the value are defined
2028 /// (initialized).
2029 Constant *getCleanShadow(Value *V) { return getCleanShadow(OrigTy: V->getType()); }
2030
2031 /// Create a dirty shadow of a given shadow type.
2032 Constant *getPoisonedShadow(Type *ShadowTy) {
2033 assert(ShadowTy);
2034 if (isa<IntegerType>(Val: ShadowTy) || isa<VectorType>(Val: ShadowTy))
2035 return Constant::getAllOnesValue(Ty: ShadowTy);
2036 if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy)) {
2037 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
2038 getPoisonedShadow(ShadowTy: AT->getElementType()));
2039 return ConstantArray::get(T: AT, V: Vals);
2040 }
2041 if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy)) {
2042 SmallVector<Constant *, 4> Vals;
2043 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
2044 Vals.push_back(Elt: getPoisonedShadow(ShadowTy: ST->getElementType(N: i)));
2045 return ConstantStruct::get(T: ST, V: Vals);
2046 }
2047 llvm_unreachable("Unexpected shadow type");
2048 }
2049
2050 /// Create a dirty shadow for a given value.
2051 Constant *getPoisonedShadow(Value *V) {
2052 Type *ShadowTy = getShadowTy(V);
2053 if (!ShadowTy)
2054 return nullptr;
2055 return getPoisonedShadow(ShadowTy);
2056 }
2057
2058 /// Create a clean (zero) origin.
2059 Value *getCleanOrigin() { return Constant::getNullValue(Ty: MS.OriginTy); }
2060
2061 /// Get the shadow value for a given Value.
2062 ///
2063 /// This function either returns the value set earlier with setShadow,
2064 /// or extracts if from ParamTLS (for function arguments).
2065 Value *getShadow(Value *V) {
2066 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2067 if (!PropagateShadow || I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2068 return getCleanShadow(V);
2069 // For instructions the shadow is already stored in the map.
2070 Value *Shadow = ShadowMap[V];
2071 if (!Shadow) {
2072 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
2073 assert(Shadow && "No shadow for a value");
2074 }
2075 return Shadow;
2076 }
2077 // Handle fully undefined values
2078 // (partially undefined constant vectors are handled later)
2079 if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(Val: V)) {
2080 Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
2081 : getCleanShadow(V);
2082 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
2083 return AllOnes;
2084 }
2085 if (Argument *A = dyn_cast<Argument>(Val: V)) {
2086 // For arguments we compute the shadow on demand and store it in the map.
2087 Value *&ShadowPtr = ShadowMap[V];
2088 if (ShadowPtr)
2089 return ShadowPtr;
2090 Function *F = A->getParent();
2091 IRBuilder<> EntryIRB(FnPrologueEnd);
2092 unsigned ArgOffset = 0;
2093 const DataLayout &DL = F->getDataLayout();
2094 for (auto &FArg : F->args()) {
2095 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
2096 LLVM_DEBUG(dbgs() << (FArg.getType()->isScalableTy()
2097 ? "vscale not fully supported\n"
2098 : "Arg is not sized\n"));
2099 if (A == &FArg) {
2100 ShadowPtr = getCleanShadow(V);
2101 setOrigin(V: A, Origin: getCleanOrigin());
2102 break;
2103 }
2104 continue;
2105 }
2106
2107 unsigned Size = FArg.hasByValAttr()
2108 ? DL.getTypeAllocSize(Ty: FArg.getParamByValType())
2109 : DL.getTypeAllocSize(Ty: FArg.getType());
2110
2111 if (A == &FArg) {
2112 bool Overflow = ArgOffset + Size > kParamTLSSize;
2113 if (FArg.hasByValAttr()) {
2114 // ByVal pointer itself has clean shadow. We copy the actual
2115 // argument shadow to the underlying memory.
2116 // Figure out maximal valid memcpy alignment.
2117 const Align ArgAlign = DL.getValueOrABITypeAlignment(
2118 Alignment: FArg.getParamAlign(), Ty: FArg.getParamByValType());
2119 Value *CpShadowPtr, *CpOriginPtr;
2120 std::tie(args&: CpShadowPtr, args&: CpOriginPtr) =
2121 getShadowOriginPtr(Addr: V, IRB&: EntryIRB, ShadowTy: EntryIRB.getInt8Ty(), Alignment: ArgAlign,
2122 /*isStore*/ true);
2123 if (!PropagateShadow || Overflow) {
2124 // ParamTLS overflow.
2125 EntryIRB.CreateMemSet(
2126 Ptr: CpShadowPtr, Val: Constant::getNullValue(Ty: EntryIRB.getInt8Ty()),
2127 Size, Align: ArgAlign);
2128 } else {
2129 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
2130 const Align CopyAlign = std::min(a: ArgAlign, b: kShadowTLSAlignment);
2131 [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
2132 Dst: CpShadowPtr, DstAlign: CopyAlign, Src: Base, SrcAlign: CopyAlign, Size);
2133 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
2134
2135 if (MS.TrackOrigins) {
2136 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
2137 // FIXME: OriginSize should be:
2138 // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
2139 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
2140 EntryIRB.CreateMemCpy(
2141 Dst: CpOriginPtr,
2142 /* by getShadowOriginPtr */ DstAlign: kMinOriginAlignment, Src: OriginPtr,
2143 /* by origin_tls[ArgOffset] */ SrcAlign: kMinOriginAlignment,
2144 Size: OriginSize);
2145 }
2146 }
2147 }
2148
2149 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2150 (MS.EagerChecks && FArg.hasAttribute(Kind: Attribute::NoUndef))) {
2151 ShadowPtr = getCleanShadow(V);
2152 setOrigin(V: A, Origin: getCleanOrigin());
2153 } else {
2154 // Shadow over TLS
2155 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
2156 ShadowPtr = EntryIRB.CreateAlignedLoad(Ty: getShadowTy(V: &FArg), Ptr: Base,
2157 Align: kShadowTLSAlignment);
2158 if (MS.TrackOrigins) {
2159 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
2160 setOrigin(V: A, Origin: EntryIRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
2161 }
2162 }
2163 LLVM_DEBUG(dbgs()
2164 << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
2165 break;
2166 }
2167
2168 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
2169 }
2170 assert(ShadowPtr && "Could not find shadow for an argument");
2171 return ShadowPtr;
2172 }
2173
2174 // Check for partially-undefined constant vectors
2175 // TODO: scalable vectors (this is hard because we do not have IRBuilder)
2176 if (isa<FixedVectorType>(Val: V->getType()) && isa<Constant>(Val: V) &&
2177 cast<Constant>(Val: V)->containsUndefOrPoisonElement() && PropagateShadow &&
2178 PoisonUndefVectors) {
2179 unsigned NumElems = cast<FixedVectorType>(Val: V->getType())->getNumElements();
2180 SmallVector<Constant *, 32> ShadowVector(NumElems);
2181 for (unsigned i = 0; i != NumElems; ++i) {
2182 Constant *Elem = cast<Constant>(Val: V)->getAggregateElement(Elt: i);
2183 ShadowVector[i] = isa<UndefValue>(Val: Elem) ? getPoisonedShadow(V: Elem)
2184 : getCleanShadow(V: Elem);
2185 }
2186
2187 Value *ShadowConstant = ConstantVector::get(V: ShadowVector);
2188 LLVM_DEBUG(dbgs() << "Partial undef constant vector: " << *V << " ==> "
2189 << *ShadowConstant << "\n");
2190
2191 return ShadowConstant;
2192 }
2193
2194 // TODO: partially-undefined constant arrays, structures, and nested types
2195
2196 // For everything else the shadow is zero.
2197 return getCleanShadow(V);
2198 }
2199
2200 /// Get the shadow for i-th argument of the instruction I.
2201 Value *getShadow(Instruction *I, int i) {
2202 return getShadow(V: I->getOperand(i));
2203 }
2204
2205 /// Get the origin for a value.
2206 Value *getOrigin(Value *V) {
2207 if (!MS.TrackOrigins)
2208 return nullptr;
2209 if (!PropagateShadow || isa<Constant>(Val: V) || isa<InlineAsm>(Val: V))
2210 return getCleanOrigin();
2211 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2212 "Unexpected value type in getOrigin()");
2213 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2214 if (I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2215 return getCleanOrigin();
2216 }
2217 Value *Origin = OriginMap[V];
2218 assert(Origin && "Missing origin");
2219 return Origin;
2220 }
2221
2222 /// Get the origin for i-th argument of the instruction I.
2223 Value *getOrigin(Instruction *I, int i) {
2224 return getOrigin(V: I->getOperand(i));
2225 }
2226
2227 /// Remember the place where a shadow check should be inserted.
2228 ///
2229 /// This location will be later instrumented with a check that will print a
2230 /// UMR warning in runtime if the shadow value is not 0.
2231 void insertCheckShadow(Value *Shadow, Value *Origin, Instruction *OrigIns) {
2232 assert(Shadow);
2233 if (!InsertChecks)
2234 return;
2235
2236 if (!DebugCounter::shouldExecute(Counter&: DebugInsertCheck)) {
2237 LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
2238 << *OrigIns << "\n");
2239 return;
2240 }
2241
2242 Type *ShadowTy = Shadow->getType();
2243 if (isScalableNonVectorType(Ty: ShadowTy)) {
2244 LLVM_DEBUG(dbgs() << "Skipping check of scalable non-vector " << *Shadow
2245 << " before " << *OrigIns << "\n");
2246 return;
2247 }
2248#ifndef NDEBUG
2249 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2250 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2251 "Can only insert checks for integer, vector, and aggregate shadow "
2252 "types");
2253#endif
2254 InstrumentationList.push_back(
2255 Elt: ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2256 }
2257
2258 /// Get shadow for value, and remember the place where a shadow check should
2259 /// be inserted.
2260 ///
2261 /// This location will be later instrumented with a check that will print a
2262 /// UMR warning in runtime if the value is not fully defined.
2263 void insertCheckShadowOf(Value *Val, Instruction *OrigIns) {
2264 assert(Val);
2265 Value *Shadow, *Origin;
2266 if (ClCheckConstantShadow) {
2267 Shadow = getShadow(V: Val);
2268 if (!Shadow)
2269 return;
2270 Origin = getOrigin(V: Val);
2271 } else {
2272 Shadow = dyn_cast_or_null<Instruction>(Val: getShadow(V: Val));
2273 if (!Shadow)
2274 return;
2275 Origin = dyn_cast_or_null<Instruction>(Val: getOrigin(V: Val));
2276 }
2277 insertCheckShadow(Shadow, Origin, OrigIns);
2278 }
2279
2280 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
2281 switch (a) {
2282 case AtomicOrdering::NotAtomic:
2283 return AtomicOrdering::NotAtomic;
2284 case AtomicOrdering::Unordered:
2285 case AtomicOrdering::Monotonic:
2286 case AtomicOrdering::Release:
2287 return AtomicOrdering::Release;
2288 case AtomicOrdering::Acquire:
2289 case AtomicOrdering::AcquireRelease:
2290 return AtomicOrdering::AcquireRelease;
2291 case AtomicOrdering::SequentiallyConsistent:
2292 return AtomicOrdering::SequentiallyConsistent;
2293 }
2294 llvm_unreachable("Unknown ordering");
2295 }
2296
2297 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2298 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2299 uint32_t OrderingTable[NumOrderings] = {};
2300
2301 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2302 OrderingTable[(int)AtomicOrderingCABI::release] =
2303 (int)AtomicOrderingCABI::release;
2304 OrderingTable[(int)AtomicOrderingCABI::consume] =
2305 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2306 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2307 (int)AtomicOrderingCABI::acq_rel;
2308 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2309 (int)AtomicOrderingCABI::seq_cst;
2310
2311 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2312 }
2313
2314 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
2315 switch (a) {
2316 case AtomicOrdering::NotAtomic:
2317 return AtomicOrdering::NotAtomic;
2318 case AtomicOrdering::Unordered:
2319 case AtomicOrdering::Monotonic:
2320 case AtomicOrdering::Acquire:
2321 return AtomicOrdering::Acquire;
2322 case AtomicOrdering::Release:
2323 case AtomicOrdering::AcquireRelease:
2324 return AtomicOrdering::AcquireRelease;
2325 case AtomicOrdering::SequentiallyConsistent:
2326 return AtomicOrdering::SequentiallyConsistent;
2327 }
2328 llvm_unreachable("Unknown ordering");
2329 }
2330
2331 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2332 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2333 uint32_t OrderingTable[NumOrderings] = {};
2334
2335 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2336 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2337 OrderingTable[(int)AtomicOrderingCABI::consume] =
2338 (int)AtomicOrderingCABI::acquire;
2339 OrderingTable[(int)AtomicOrderingCABI::release] =
2340 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2341 (int)AtomicOrderingCABI::acq_rel;
2342 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2343 (int)AtomicOrderingCABI::seq_cst;
2344
2345 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2346 }
2347
2348 // ------------------- Visitors.
2349 using InstVisitor<MemorySanitizerVisitor>::visit;
2350 void visit(Instruction &I) {
2351 if (I.getMetadata(KindID: LLVMContext::MD_nosanitize))
2352 return;
2353 // Don't want to visit if we're in the prologue
2354 if (isInPrologue(I))
2355 return;
2356 if (!DebugCounter::shouldExecute(Counter&: DebugInstrumentInstruction)) {
2357 LLVM_DEBUG(dbgs() << "Skipping instruction: " << I << "\n");
2358 // We still need to set the shadow and origin to clean values.
2359 setShadow(V: &I, SV: getCleanShadow(V: &I));
2360 setOrigin(V: &I, Origin: getCleanOrigin());
2361 return;
2362 }
2363
2364 Instructions.push_back(Elt: &I);
2365 }
2366
2367 /// Instrument LoadInst
2368 ///
2369 /// Loads the corresponding shadow and (optionally) origin.
2370 /// Optionally, checks that the load address is fully defined.
2371 void visitLoadInst(LoadInst &I) {
2372 assert(I.getType()->isSized() && "Load type must have size");
2373 assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2374 NextNodeIRBuilder IRB(&I);
2375 Type *ShadowTy = getShadowTy(V: &I);
2376 Value *Addr = I.getPointerOperand();
2377 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2378 const Align Alignment = I.getAlign();
2379 if (PropagateShadow) {
2380 std::tie(args&: ShadowPtr, args&: OriginPtr) =
2381 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2382 setShadow(V: &I,
2383 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2384 } else {
2385 setShadow(V: &I, SV: getCleanShadow(V: &I));
2386 }
2387
2388 if (ClCheckAccessAddress)
2389 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2390
2391 if (I.isAtomic())
2392 I.setOrdering(addAcquireOrdering(a: I.getOrdering()));
2393
2394 if (MS.TrackOrigins) {
2395 if (PropagateShadow) {
2396 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
2397 setOrigin(
2398 V: &I, Origin: IRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: OriginPtr, Align: OriginAlignment));
2399 } else {
2400 setOrigin(V: &I, Origin: getCleanOrigin());
2401 }
2402 }
2403 }
2404
2405 /// Instrument StoreInst
2406 ///
2407 /// Stores the corresponding shadow and (optionally) origin.
2408 /// Optionally, checks that the store address is fully defined.
2409 void visitStoreInst(StoreInst &I) {
2410 StoreList.push_back(Elt: &I);
2411 if (ClCheckAccessAddress)
2412 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2413 }
2414
2415 void handleCASOrRMW(Instruction &I) {
2416 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2417
2418 IRBuilder<> IRB(&I);
2419 Value *Addr = I.getOperand(i: 0);
2420 Value *Val = I.getOperand(i: 1);
2421 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, ShadowTy: getShadowTy(V: Val), Alignment: Align(1),
2422 /*isStore*/ true)
2423 .first;
2424
2425 if (ClCheckAccessAddress)
2426 insertCheckShadowOf(Val: Addr, OrigIns: &I);
2427
2428 // Only test the conditional argument of cmpxchg instruction.
2429 // The other argument can potentially be uninitialized, but we can not
2430 // detect this situation reliably without possible false positives.
2431 if (isa<AtomicCmpXchgInst>(Val: I))
2432 insertCheckShadowOf(Val, OrigIns: &I);
2433
2434 IRB.CreateStore(Val: getCleanShadow(V: Val), Ptr: ShadowPtr);
2435
2436 setShadow(V: &I, SV: getCleanShadow(V: &I));
2437 setOrigin(V: &I, Origin: getCleanOrigin());
2438 }
2439
2440 void visitAtomicRMWInst(AtomicRMWInst &I) {
2441 handleCASOrRMW(I);
2442 I.setOrdering(addReleaseOrdering(a: I.getOrdering()));
2443 }
2444
2445 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2446 handleCASOrRMW(I);
2447 I.setSuccessOrdering(addReleaseOrdering(a: I.getSuccessOrdering()));
2448 }
2449
2450 /// Generic handler to compute shadow for == and != comparisons.
2451 ///
2452 /// This function is used by handleEqualityComparison and visitSwitchInst.
2453 ///
2454 /// Sometimes the comparison result is known even if some of the bits of the
2455 /// arguments are not.
2456 Value *propagateEqualityComparison(IRBuilder<> &IRB, Value *A, Value *B,
2457 Value *Sa, Value *Sb) {
2458 assert(getShadowTy(A) == Sa->getType());
2459 assert(getShadowTy(B) == Sb->getType());
2460
2461 // Get rid of pointers and vectors of pointers.
2462 // For ints (and vectors of ints), types of A and Sa match,
2463 // and this is a no-op.
2464 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2465 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2466
2467 // A == B <==> (C = A^B) == 0
2468 // A != B <==> (C = A^B) != 0
2469 // Sc = Sa | Sb
2470 Value *C = IRB.CreateXor(LHS: A, RHS: B);
2471 Value *Sc = IRB.CreateOr(LHS: Sa, RHS: Sb);
2472 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2473 // Result is defined if one of the following is true
2474 // * there is a defined 1 bit in C
2475 // * C is fully defined
2476 // Si = !(C & ~Sc) && Sc
2477 Value *Zero = Constant::getNullValue(Ty: Sc->getType());
2478 Value *MinusOne = Constant::getAllOnesValue(Ty: Sc->getType());
2479 Value *LHS = IRB.CreateICmpNE(LHS: Sc, RHS: Zero);
2480 Value *RHS =
2481 IRB.CreateICmpEQ(LHS: IRB.CreateAnd(LHS: IRB.CreateXor(LHS: Sc, RHS: MinusOne), RHS: C), RHS: Zero);
2482 Value *Si = IRB.CreateAnd(LHS, RHS);
2483 Si->setName("_msprop_icmp");
2484
2485 return Si;
2486 }
2487
2488 // Instrument:
2489 // switch i32 %Val, label %else [ i32 0, label %A
2490 // i32 1, label %B
2491 // i32 2, label %C ]
2492 //
2493 // Typically, the switch input value (%Val) is fully initialized.
2494 //
2495 // Sometimes the compiler may convert (icmp + br) into a switch statement.
2496 // MSan allows icmp eq/ne with partly initialized inputs to still result in a
2497 // fully initialized output, if there exists a bit that is initialized in
2498 // both inputs with a differing value. For compatibility, we support this in
2499 // the switch instrumentation as well. Note that this edge case only applies
2500 // if the switch input value does not match *any* of the cases (matching any
2501 // of the cases requires an exact, fully initialized match).
2502 //
2503 // ShadowCases = 0
2504 // | propagateEqualityComparison(Val, 0)
2505 // | propagateEqualityComparison(Val, 1)
2506 // | propagateEqualityComparison(Val, 2))
2507 void visitSwitchInst(SwitchInst &SI) {
2508 IRBuilder<> IRB(&SI);
2509
2510 Value *Val = SI.getCondition();
2511 Value *ShadowVal = getShadow(V: Val);
2512 // TODO: add fast path - if the condition is fully initialized, we know
2513 // there is no UUM, without needing to consider the case values below.
2514
2515 // Some code (e.g., AMDGPUGenMCCodeEmitter.inc) has tens of thousands of
2516 // cases. This results in an extremely long chained expression for MSan's
2517 // switch instrumentation, which can cause the JumpThreadingPass to have a
2518 // stack overflow or excessive runtime. We limit the number of cases
2519 // considered, with the tradeoff of niche false negatives.
2520 // TODO: figure out a better solution.
2521 int casesToConsider = ClSwitchPrecision;
2522
2523 Value *ShadowCases = nullptr;
2524 for (auto Case : SI.cases()) {
2525 if (casesToConsider <= 0)
2526 break;
2527
2528 Value *Comparator = Case.getCaseValue();
2529 // TODO: some simplification is possible when comparing multiple cases
2530 // simultaneously.
2531 Value *ComparisonShadow = propagateEqualityComparison(
2532 IRB, A: Val, B: Comparator, Sa: ShadowVal, Sb: getShadow(V: Comparator));
2533
2534 if (ShadowCases)
2535 ShadowCases = IRB.CreateOr(LHS: ShadowCases, RHS: ComparisonShadow);
2536 else
2537 ShadowCases = ComparisonShadow;
2538
2539 casesToConsider--;
2540 }
2541
2542 if (ShadowCases)
2543 insertCheckShadow(Shadow: ShadowCases, Origin: getOrigin(V: Val), OrigIns: &SI);
2544 }
2545
2546 // Vector manipulation.
2547 void visitExtractElementInst(ExtractElementInst &I) {
2548 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 1), OrigIns: &I);
2549 IRBuilder<> IRB(&I);
2550 setShadow(V: &I, SV: IRB.CreateExtractElement(Vec: getShadow(I: &I, i: 0), Idx: I.getOperand(i_nocapture: 1),
2551 Name: "_msprop"));
2552 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2553 }
2554
2555 void visitInsertElementInst(InsertElementInst &I) {
2556 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 2), OrigIns: &I);
2557 IRBuilder<> IRB(&I);
2558 auto *Shadow0 = getShadow(I: &I, i: 0);
2559 auto *Shadow1 = getShadow(I: &I, i: 1);
2560 setShadow(V: &I, SV: IRB.CreateInsertElement(Vec: Shadow0, NewElt: Shadow1, Idx: I.getOperand(i_nocapture: 2),
2561 Name: "_msprop"));
2562 setOriginForNaryOp(I);
2563 }
2564
2565 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2566 IRBuilder<> IRB(&I);
2567 auto *Shadow0 = getShadow(I: &I, i: 0);
2568 auto *Shadow1 = getShadow(I: &I, i: 1);
2569 setShadow(V: &I, SV: IRB.CreateShuffleVector(V1: Shadow0, V2: Shadow1, Mask: I.getShuffleMask(),
2570 Name: "_msprop"));
2571 setOriginForNaryOp(I);
2572 }
2573
2574 // Casts.
2575 void visitSExtInst(SExtInst &I) {
2576 IRBuilder<> IRB(&I);
2577 setShadow(V: &I, SV: IRB.CreateSExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2578 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2579 }
2580
2581 void visitZExtInst(ZExtInst &I) {
2582 IRBuilder<> IRB(&I);
2583 setShadow(V: &I, SV: IRB.CreateZExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2584 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2585 }
2586
2587 void visitTruncInst(TruncInst &I) {
2588 IRBuilder<> IRB(&I);
2589 setShadow(V: &I, SV: IRB.CreateTrunc(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2590 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2591 }
2592
2593 void visitBitCastInst(BitCastInst &I) {
2594 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2595 // a musttail call and a ret, don't instrument. New instructions are not
2596 // allowed after a musttail call.
2597 if (auto *CI = dyn_cast<CallInst>(Val: I.getOperand(i_nocapture: 0)))
2598 if (CI->isMustTailCall())
2599 return;
2600 IRBuilder<> IRB(&I);
2601 setShadow(V: &I, SV: IRB.CreateBitCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I)));
2602 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2603 }
2604
2605 void visitPtrToIntInst(PtrToIntInst &I) {
2606 IRBuilder<> IRB(&I);
2607 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2608 Name: "_msprop_ptrtoint"));
2609 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2610 }
2611
2612 void visitIntToPtrInst(IntToPtrInst &I) {
2613 IRBuilder<> IRB(&I);
2614 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2615 Name: "_msprop_inttoptr"));
2616 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2617 }
2618
2619 /// Handle LLVM and NEON vector convert intrinsics.
2620 ///
2621 /// e.g., <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>)
2622 /// i32 @llvm.aarch64.neon.fcvtms.i32.f64 (double)
2623 /// <2 x i32> @fptoui (<2 x float>)
2624 /// i64 @llvm.fptosi.sat.i64.f64(double)
2625 ///
2626 /// Note that the size of input/output elements can differ e.g.,
2627 /// double @sitofp(i32)
2628 /// but the number of elements must be the same.
2629 ///
2630 /// For conversions to or from fixed-point, there is a trailing argument to
2631 /// indicate the fixed-point precision:
2632 /// - <4 x float> llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32>, i32)
2633 /// - <4 x i32> llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float>, i32)
2634 ///
2635 /// For x86 SSE vector convert intrinsics, see
2636 /// handleSSEVectorConvertIntrinsic().
2637 void handleGenericVectorConvertIntrinsic(Instruction &I, bool FixedPoint) {
2638 [[maybe_unused]] unsigned NumArgs = I.getNumOperands();
2639 if (auto *CI = dyn_cast<CallInst>(Val: &I))
2640 NumArgs = CI->arg_size();
2641
2642 if (FixedPoint) {
2643 assert(NumArgs == 2);
2644 Value *Precision = I.getOperand(i: 1);
2645 insertCheckShadowOf(Val: Precision, OrigIns: &I);
2646 } else {
2647 assert(NumArgs == 1);
2648 }
2649
2650 IRBuilder<> IRB(&I);
2651 Value *S0 = getShadow(I: &I, i: 0);
2652
2653 /// For scalars:
2654 /// Since they are converting from floating-point to integer, or between
2655 /// different width floating-point values, the output is:
2656 /// - fully uninitialized if *any* bit of the input is uninitialized
2657 /// - fully ininitialized if all bits of the input are ininitialized
2658 /// We apply the same principle on a per-field basis for vectors.
2659 Value *OutShadow = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)),
2660 DestTy: getShadowTy(V: &I));
2661 setShadow(V: &I, SV: OutShadow);
2662 setOriginForNaryOp(I);
2663 }
2664
2665 void visitFPToSIInst(CastInst &I) {
2666 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2667 }
2668 void visitFPToUIInst(CastInst &I) {
2669 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2670 }
2671 void visitSIToFPInst(CastInst &I) {
2672 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2673 }
2674 void visitUIToFPInst(CastInst &I) {
2675 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2676 }
2677
2678 void visitFPExtInst(CastInst &I) {
2679 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2680 }
2681 void visitFPTruncInst(CastInst &I) {
2682 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2683 }
2684
2685 /// Generic handler to compute shadow for bitwise AND.
2686 ///
2687 /// This is used by 'visitAnd' but also as a primitive for other handlers.
2688 ///
2689 /// This code is precise: it implements the rule that "And" of an initialized
2690 /// zero bit always results in an initialized value:
2691 // 1&1 => 1; 0&1 => 0; p&1 => p;
2692 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2693 // 1&p => p; 0&p => 0; p&p => p;
2694 //
2695 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2696 Value *handleBitwiseAnd(IRBuilder<> &IRB, Value *V1, Value *V2, Value *S1,
2697 Value *S2) {
2698 // "The two arguments to the ‘and’ instruction must be integer or vector
2699 // of integer values. Both arguments must have identical types."
2700 //
2701 // We enforce this condition for all callers to handleBitwiseAnd(); callers
2702 // with non-integer types should call CreateAppToShadowCast() themselves.
2703 assert(V1->getType()->isIntOrIntVectorTy());
2704 assert(V1->getType() == V2->getType());
2705
2706 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2707 assert(V1->getType() == S1->getType());
2708 assert(V2->getType() == S2->getType());
2709
2710 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2711 Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2712 Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2713
2714 return IRB.CreateOr(Ops: {S1S2, V1S2, S1V2});
2715 }
2716
2717 /// Handler for bitwise AND operator.
2718 void visitAnd(BinaryOperator &I) {
2719 IRBuilder<> IRB(&I);
2720 Value *V1 = I.getOperand(i_nocapture: 0);
2721 Value *V2 = I.getOperand(i_nocapture: 1);
2722 Value *S1 = getShadow(I: &I, i: 0);
2723 Value *S2 = getShadow(I: &I, i: 1);
2724
2725 Value *OutShadow = handleBitwiseAnd(IRB, V1, V2, S1, S2);
2726
2727 setShadow(V: &I, SV: OutShadow);
2728 setOriginForNaryOp(I);
2729 }
2730
2731 void visitOr(BinaryOperator &I) {
2732 IRBuilder<> IRB(&I);
2733 // "Or" of 1 and a poisoned value results in unpoisoned value:
2734 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2735 // 1|0 => 1; 0|0 => 0; p|0 => p;
2736 // 1|p => 1; 0|p => p; p|p => p;
2737 //
2738 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2739 //
2740 // If the "disjoint OR" property is violated, the result is poison, and
2741 // hence the entire shadow is uninitialized:
2742 // S = S | SignExt(V1 & V2 != 0)
2743 Value *S1 = getShadow(I: &I, i: 0);
2744 Value *S2 = getShadow(I: &I, i: 1);
2745 Value *V1 = I.getOperand(i_nocapture: 0);
2746 Value *V2 = I.getOperand(i_nocapture: 1);
2747
2748 // "The two arguments to the ‘or’ instruction must be integer or vector
2749 // of integer values. Both arguments must have identical types."
2750 assert(V1->getType()->isIntOrIntVectorTy());
2751 assert(V1->getType() == V2->getType());
2752
2753 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2754 assert(V1->getType() == S1->getType());
2755 assert(V2->getType() == S2->getType());
2756
2757 Value *NotV1 = IRB.CreateNot(V: V1);
2758 Value *NotV2 = IRB.CreateNot(V: V2);
2759
2760 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2761 Value *S2NotV1 = IRB.CreateAnd(LHS: NotV1, RHS: S2);
2762 Value *S1NotV2 = IRB.CreateAnd(LHS: S1, RHS: NotV2);
2763
2764 Value *S = IRB.CreateOr(Ops: {S1S2, S2NotV1, S1NotV2});
2765
2766 if (ClPreciseDisjointOr && cast<PossiblyDisjointInst>(Val: &I)->isDisjoint()) {
2767 Value *V1V2 = IRB.CreateAnd(LHS: V1, RHS: V2);
2768 Value *DisjointOrShadow = IRB.CreateSExt(
2769 V: IRB.CreateICmpNE(LHS: V1V2, RHS: getCleanShadow(V: V1V2)), DestTy: V1V2->getType());
2770 S = IRB.CreateOr(LHS: S, RHS: DisjointOrShadow, Name: "_ms_disjoint");
2771 }
2772
2773 setShadow(V: &I, SV: S);
2774 setOriginForNaryOp(I);
2775 }
2776
2777 /// Default propagation of shadow and/or origin.
2778 ///
2779 /// This class implements the general case of shadow propagation, used in all
2780 /// cases where we don't know and/or don't care about what the operation
2781 /// actually does. It converts all input shadow values to a common type
2782 /// (extending or truncating as necessary), and bitwise OR's them.
2783 ///
2784 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2785 /// fully initialized), and less prone to false positives.
2786 ///
2787 /// This class also implements the general case of origin propagation. For a
2788 /// Nary operation, result origin is set to the origin of an argument that is
2789 /// not entirely initialized. If there is more than one such arguments, the
2790 /// rightmost of them is picked. It does not matter which one is picked if all
2791 /// arguments are initialized.
2792 template <bool CombineShadow> class Combiner {
2793 Value *Shadow = nullptr;
2794 Value *Origin = nullptr;
2795 IRBuilder<> &IRB;
2796 MemorySanitizerVisitor *MSV;
2797
2798 public:
2799 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2800 : IRB(IRB), MSV(MSV) {}
2801
2802 /// Add a pair of shadow and origin values to the mix.
2803 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2804 if (CombineShadow) {
2805 assert(OpShadow);
2806 if (!Shadow)
2807 Shadow = OpShadow;
2808 else {
2809 OpShadow = MSV->CreateShadowCast(IRB, V: OpShadow, dstTy: Shadow->getType());
2810 Shadow = IRB.CreateOr(LHS: Shadow, RHS: OpShadow, Name: "_msprop");
2811 }
2812 }
2813
2814 if (MSV->MS.TrackOrigins) {
2815 assert(OpOrigin);
2816 if (!Origin) {
2817 Origin = OpOrigin;
2818 } else {
2819 Constant *ConstOrigin = dyn_cast<Constant>(Val: OpOrigin);
2820 // No point in adding something that might result in 0 origin value.
2821 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2822 Value *Cond = MSV->convertToBool(V: OpShadow, IRB);
2823 Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2824 }
2825 }
2826 }
2827 return *this;
2828 }
2829
2830 /// Add an application value to the mix.
2831 Combiner &Add(Value *V) {
2832 Value *OpShadow = MSV->getShadow(V);
2833 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2834 return Add(OpShadow, OpOrigin);
2835 }
2836
2837 /// Set the current combined values as the given instruction's shadow
2838 /// and origin.
2839 void Done(Instruction *I) {
2840 if (CombineShadow) {
2841 assert(Shadow);
2842 Shadow = MSV->CreateShadowCast(IRB, V: Shadow, dstTy: MSV->getShadowTy(V: I));
2843 MSV->setShadow(V: I, SV: Shadow);
2844 }
2845 if (MSV->MS.TrackOrigins) {
2846 assert(Origin);
2847 MSV->setOrigin(V: I, Origin);
2848 }
2849 }
2850
2851 /// Store the current combined value at the specified origin
2852 /// location.
2853 void DoneAndStoreOrigin(TypeSize TS, Value *OriginPtr) {
2854 if (MSV->MS.TrackOrigins) {
2855 assert(Origin);
2856 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, Alignment: kMinOriginAlignment);
2857 }
2858 }
2859 };
2860
2861 using ShadowAndOriginCombiner = Combiner<true>;
2862 using OriginCombiner = Combiner<false>;
2863
2864 /// Propagate origin for arbitrary operation.
2865 void setOriginForNaryOp(Instruction &I) {
2866 if (!MS.TrackOrigins)
2867 return;
2868 IRBuilder<> IRB(&I);
2869 OriginCombiner OC(this, IRB);
2870 for (Use &Op : I.operands())
2871 OC.Add(V: Op.get());
2872 OC.Done(I: &I);
2873 }
2874
2875 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2876 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2877 "Vector of pointers is not a valid shadow type");
2878 return Ty->isVectorTy() ? cast<FixedVectorType>(Val: Ty)->getNumElements() *
2879 Ty->getScalarSizeInBits()
2880 : Ty->getPrimitiveSizeInBits();
2881 }
2882
2883 /// Cast between two shadow types, extending or truncating as
2884 /// necessary.
2885 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2886 bool Signed = false) {
2887 Type *srcTy = V->getType();
2888 if (srcTy == dstTy)
2889 return V;
2890 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: srcTy);
2891 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: dstTy);
2892 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2893 return IRB.CreateICmpNE(LHS: V, RHS: getCleanShadow(V));
2894
2895 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2896 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2897 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2898 cast<VectorType>(Val: dstTy)->getElementCount() ==
2899 cast<VectorType>(Val: srcTy)->getElementCount())
2900 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2901 Value *V1 = IRB.CreateBitCast(V, DestTy: Type::getIntNTy(C&: *MS.C, N: srcSizeInBits));
2902 Value *V2 =
2903 IRB.CreateIntCast(V: V1, DestTy: Type::getIntNTy(C&: *MS.C, N: dstSizeInBits), isSigned: Signed);
2904 return IRB.CreateBitCast(V: V2, DestTy: dstTy);
2905 // TODO: handle struct types.
2906 }
2907
2908 /// Cast an application value to the type of its own shadow.
2909 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2910 Type *ShadowTy = getShadowTy(V);
2911 if (V->getType() == ShadowTy)
2912 return V;
2913 if (V->getType()->isPtrOrPtrVectorTy())
2914 return IRB.CreatePtrToInt(V, DestTy: ShadowTy);
2915 else
2916 return IRB.CreateBitCast(V, DestTy: ShadowTy);
2917 }
2918
2919 /// Propagate shadow for arbitrary operation.
2920 void handleShadowOr(Instruction &I) {
2921 IRBuilder<> IRB(&I);
2922 ShadowAndOriginCombiner SC(this, IRB);
2923 for (Use &Op : I.operands())
2924 SC.Add(V: Op.get());
2925 SC.Done(I: &I);
2926 }
2927
2928 // Perform a bitwise OR on the horizontal pairs (or other specified grouping)
2929 // of elements.
2930 //
2931 // For example, suppose we have:
2932 // VectorA: <a0, a1, a2, a3, a4, a5>
2933 // VectorB: <b0, b1, b2, b3, b4, b5>
2934 // ReductionFactor: 3
2935 // Shards: 1
2936 // The output would be:
2937 // <a0|a1|a2, a3|a4|a5, b0|b1|b2, b3|b4|b5>
2938 //
2939 // If we have:
2940 // VectorA: <a0, a1, a2, a3, a4, a5, a6, a7>
2941 // VectorB: <b0, b1, b2, b3, b4, b5, b6, b7>
2942 // ReductionFactor: 2
2943 // Shards: 2
2944 // then a and be each have 2 "shards", resulting in the output being
2945 // interleaved:
2946 // <a0|a1, a2|a3, b0|b1, b2|b3, a4|a5, a6|a7, b4|b5, b6|b7>
2947 //
2948 // This is convenient for instrumenting horizontal add/sub.
2949 // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic().
2950 Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor,
2951 unsigned Shards, Value *VectorA, Value *VectorB) {
2952 assert(isa<FixedVectorType>(VectorA->getType()));
2953 unsigned NumElems =
2954 cast<FixedVectorType>(Val: VectorA->getType())->getNumElements();
2955
2956 [[maybe_unused]] unsigned TotalNumElems = NumElems;
2957 if (VectorB) {
2958 assert(VectorA->getType() == VectorB->getType());
2959 TotalNumElems *= 2;
2960 }
2961
2962 assert(NumElems % (ReductionFactor * Shards) == 0);
2963
2964 Value *Or = nullptr;
2965
2966 IRBuilder<> IRB(&I);
2967 for (unsigned i = 0; i < ReductionFactor; i++) {
2968 SmallVector<int, 16> Mask;
2969
2970 for (unsigned j = 0; j < Shards; j++) {
2971 unsigned Offset = NumElems / Shards * j;
2972
2973 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2974 Mask.push_back(Elt: Offset + X + i);
2975
2976 if (VectorB) {
2977 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2978 Mask.push_back(Elt: NumElems + Offset + X + i);
2979 }
2980 }
2981
2982 Value *Masked;
2983 if (VectorB)
2984 Masked = IRB.CreateShuffleVector(V1: VectorA, V2: VectorB, Mask);
2985 else
2986 Masked = IRB.CreateShuffleVector(V: VectorA, Mask);
2987
2988 if (Or)
2989 Or = IRB.CreateOr(LHS: Or, RHS: Masked);
2990 else
2991 Or = Masked;
2992 }
2993
2994 return Or;
2995 }
2996
2997 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
2998 /// fields.
2999 ///
3000 /// e.g., <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16>)
3001 /// <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8>, <16 x i8>)
3002 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards) {
3003 assert(I.arg_size() == 1 || I.arg_size() == 2);
3004
3005 assert(I.getType()->isVectorTy());
3006 assert(I.getArgOperand(0)->getType()->isVectorTy());
3007
3008 [[maybe_unused]] FixedVectorType *ParamType =
3009 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
3010 assert((I.arg_size() != 2) ||
3011 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
3012 [[maybe_unused]] FixedVectorType *ReturnType =
3013 cast<FixedVectorType>(Val: I.getType());
3014 assert(ParamType->getNumElements() * I.arg_size() ==
3015 2 * ReturnType->getNumElements());
3016
3017 IRBuilder<> IRB(&I);
3018
3019 // Horizontal OR of shadow
3020 Value *FirstArgShadow = getShadow(I: &I, i: 0);
3021 Value *SecondArgShadow = nullptr;
3022 if (I.arg_size() == 2)
3023 SecondArgShadow = getShadow(I: &I, i: 1);
3024
3025 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
3026 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
3027
3028 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
3029
3030 setShadow(V: &I, SV: OrShadow);
3031 setOriginForNaryOp(I);
3032 }
3033
3034 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
3035 /// fields, with the parameters reinterpreted to have elements of a specified
3036 /// width. For example:
3037 /// @llvm.x86.ssse3.phadd.w(<1 x i64> [[VAR1]], <1 x i64> [[VAR2]])
3038 /// conceptually operates on
3039 /// (<4 x i16> [[VAR1]], <4 x i16> [[VAR2]])
3040 /// and can be handled with ReinterpretElemWidth == 16.
3041 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards,
3042 int ReinterpretElemWidth) {
3043 assert(I.arg_size() == 1 || I.arg_size() == 2);
3044
3045 assert(I.getType()->isVectorTy());
3046 assert(I.getArgOperand(0)->getType()->isVectorTy());
3047
3048 FixedVectorType *ParamType =
3049 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
3050 assert((I.arg_size() != 2) ||
3051 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
3052
3053 [[maybe_unused]] FixedVectorType *ReturnType =
3054 cast<FixedVectorType>(Val: I.getType());
3055 assert(ParamType->getNumElements() * I.arg_size() ==
3056 2 * ReturnType->getNumElements());
3057
3058 IRBuilder<> IRB(&I);
3059
3060 FixedVectorType *ReinterpretShadowTy = nullptr;
3061 assert(isAligned(Align(ReinterpretElemWidth),
3062 ParamType->getPrimitiveSizeInBits()));
3063 ReinterpretShadowTy = FixedVectorType::get(
3064 ElementType: IRB.getIntNTy(N: ReinterpretElemWidth),
3065 NumElts: ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth);
3066
3067 // Horizontal OR of shadow
3068 Value *FirstArgShadow = getShadow(I: &I, i: 0);
3069 FirstArgShadow = IRB.CreateBitCast(V: FirstArgShadow, DestTy: ReinterpretShadowTy);
3070
3071 // If we had two parameters each with an odd number of elements, the total
3072 // number of elements is even, but we have never seen this in extant
3073 // instruction sets, so we enforce that each parameter must have an even
3074 // number of elements.
3075 assert(isAligned(
3076 Align(2),
3077 cast<FixedVectorType>(FirstArgShadow->getType())->getNumElements()));
3078
3079 Value *SecondArgShadow = nullptr;
3080 if (I.arg_size() == 2) {
3081 SecondArgShadow = getShadow(I: &I, i: 1);
3082 SecondArgShadow = IRB.CreateBitCast(V: SecondArgShadow, DestTy: ReinterpretShadowTy);
3083 }
3084
3085 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
3086 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
3087
3088 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
3089
3090 setShadow(V: &I, SV: OrShadow);
3091 setOriginForNaryOp(I);
3092 }
3093
3094 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
3095
3096 // Handle multiplication by constant.
3097 //
3098 // Handle a special case of multiplication by constant that may have one or
3099 // more zeros in the lower bits. This makes corresponding number of lower bits
3100 // of the result zero as well. We model it by shifting the other operand
3101 // shadow left by the required number of bits. Effectively, we transform
3102 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
3103 // We use multiplication by 2**N instead of shift to cover the case of
3104 // multiplication by 0, which may occur in some elements of a vector operand.
3105 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
3106 Value *OtherArg) {
3107 Constant *ShadowMul;
3108 Type *Ty = ConstArg->getType();
3109 if (auto *VTy = dyn_cast<VectorType>(Val: Ty)) {
3110 unsigned NumElements = cast<FixedVectorType>(Val: VTy)->getNumElements();
3111 Type *EltTy = VTy->getElementType();
3112 SmallVector<Constant *, 16> Elements;
3113 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
3114 if (ConstantInt *Elt =
3115 dyn_cast<ConstantInt>(Val: ConstArg->getAggregateElement(Elt: Idx))) {
3116 const APInt &V = Elt->getValue();
3117 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
3118 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: V2));
3119 } else {
3120 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: 1));
3121 }
3122 }
3123 ShadowMul = ConstantVector::get(V: Elements);
3124 } else {
3125 if (ConstantInt *Elt = dyn_cast<ConstantInt>(Val: ConstArg)) {
3126 const APInt &V = Elt->getValue();
3127 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
3128 ShadowMul = ConstantInt::get(Ty, V: V2);
3129 } else {
3130 ShadowMul = ConstantInt::get(Ty, V: 1);
3131 }
3132 }
3133
3134 IRBuilder<> IRB(&I);
3135 setShadow(V: &I,
3136 SV: IRB.CreateMul(LHS: getShadow(V: OtherArg), RHS: ShadowMul, Name: "msprop_mul_cst"));
3137 setOrigin(V: &I, Origin: getOrigin(V: OtherArg));
3138 }
3139
3140 void visitMul(BinaryOperator &I) {
3141 Constant *constOp0 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0));
3142 Constant *constOp1 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3143 if (constOp0 && !constOp1)
3144 handleMulByConstant(I, ConstArg: constOp0, OtherArg: I.getOperand(i_nocapture: 1));
3145 else if (constOp1 && !constOp0)
3146 handleMulByConstant(I, ConstArg: constOp1, OtherArg: I.getOperand(i_nocapture: 0));
3147 else
3148 handleShadowOr(I);
3149 }
3150
3151 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
3152 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
3153 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
3154 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
3155 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
3156 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
3157
3158 void handleIntegerDiv(Instruction &I) {
3159 IRBuilder<> IRB(&I);
3160 // Strict on the second argument.
3161 insertCheckShadowOf(Val: I.getOperand(i: 1), OrigIns: &I);
3162 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3163 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3164 }
3165
3166 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3167 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3168 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
3169 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
3170
3171 // Floating point division is side-effect free. We can not require that the
3172 // divisor is fully initialized and must propagate shadow. See PR37523.
3173 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
3174 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
3175
3176 /// Instrument == and != comparisons.
3177 ///
3178 /// Sometimes the comparison result is known even if some of the bits of the
3179 /// arguments are not.
3180 void handleEqualityComparison(ICmpInst &I) {
3181 IRBuilder<> IRB(&I);
3182 Value *A = I.getOperand(i_nocapture: 0);
3183 Value *B = I.getOperand(i_nocapture: 1);
3184 Value *Sa = getShadow(V: A);
3185 Value *Sb = getShadow(V: B);
3186
3187 Value *Si = propagateEqualityComparison(IRB, A, B, Sa, Sb);
3188
3189 setShadow(V: &I, SV: Si);
3190 setOriginForNaryOp(I);
3191 }
3192
3193 /// Instrument relational comparisons.
3194 ///
3195 /// This function does exact shadow propagation for all relational
3196 /// comparisons of integers, pointers and vectors of those.
3197 /// FIXME: output seems suboptimal when one of the operands is a constant
3198 void handleRelationalComparisonExact(ICmpInst &I) {
3199 IRBuilder<> IRB(&I);
3200 Value *A = I.getOperand(i_nocapture: 0);
3201 Value *B = I.getOperand(i_nocapture: 1);
3202 Value *Sa = getShadow(V: A);
3203 Value *Sb = getShadow(V: B);
3204
3205 // Get rid of pointers and vectors of pointers.
3206 // For ints (and vectors of ints), types of A and Sa match,
3207 // and this is a no-op.
3208 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
3209 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
3210
3211 // Let [a0, a1] be the interval of possible values of A, taking into account
3212 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
3213 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
3214 bool IsSigned = I.isSigned();
3215
3216 auto GetMinMaxUnsigned = [&](Value *V, Value *S) {
3217 if (IsSigned) {
3218 // Sign-flip to map from signed range to unsigned range. Relation A vs B
3219 // should be preserved, if checked with `getUnsignedPredicate()`.
3220 // Relationship between Amin, Amax, Bmin, Bmax also will not be
3221 // affected, as they are created by effectively adding/substructing from
3222 // A (or B) a value, derived from shadow, with no overflow, either
3223 // before or after sign flip.
3224 APInt MinVal =
3225 APInt::getSignedMinValue(numBits: V->getType()->getScalarSizeInBits());
3226 V = IRB.CreateXor(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: MinVal));
3227 }
3228 // Minimize undefined bits.
3229 Value *Min = IRB.CreateAnd(LHS: V, RHS: IRB.CreateNot(V: S));
3230 Value *Max = IRB.CreateOr(LHS: V, RHS: S);
3231 return std::make_pair(x&: Min, y&: Max);
3232 };
3233
3234 auto [Amin, Amax] = GetMinMaxUnsigned(A, Sa);
3235 auto [Bmin, Bmax] = GetMinMaxUnsigned(B, Sb);
3236 Value *S1 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amin, RHS: Bmax);
3237 Value *S2 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amax, RHS: Bmin);
3238
3239 Value *Si = IRB.CreateXor(LHS: S1, RHS: S2);
3240 setShadow(V: &I, SV: Si);
3241 setOriginForNaryOp(I);
3242 }
3243
3244 /// Instrument signed relational comparisons.
3245 ///
3246 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
3247 /// bit of the shadow. Everything else is delegated to handleShadowOr().
3248 void handleSignedRelationalComparison(ICmpInst &I) {
3249 Constant *constOp;
3250 Value *op = nullptr;
3251 CmpInst::Predicate pre;
3252 if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3253 op = I.getOperand(i_nocapture: 0);
3254 pre = I.getPredicate();
3255 } else if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0)))) {
3256 op = I.getOperand(i_nocapture: 1);
3257 pre = I.getSwappedPredicate();
3258 } else {
3259 handleShadowOr(I);
3260 return;
3261 }
3262
3263 if ((constOp->isNullValue() &&
3264 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
3265 (constOp->isAllOnesValue() &&
3266 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
3267 IRBuilder<> IRB(&I);
3268 Value *Shadow = IRB.CreateICmpSLT(LHS: getShadow(V: op), RHS: getCleanShadow(V: op),
3269 Name: "_msprop_icmp_s");
3270 setShadow(V: &I, SV: Shadow);
3271 setOrigin(V: &I, Origin: getOrigin(V: op));
3272 } else {
3273 handleShadowOr(I);
3274 }
3275 }
3276
3277 void visitICmpInst(ICmpInst &I) {
3278 if (!ClHandleICmp) {
3279 handleShadowOr(I);
3280 return;
3281 }
3282 if (I.isEquality()) {
3283 handleEqualityComparison(I);
3284 return;
3285 }
3286
3287 assert(I.isRelational());
3288 if (ClHandleICmpExact) {
3289 handleRelationalComparisonExact(I);
3290 return;
3291 }
3292 if (I.isSigned()) {
3293 handleSignedRelationalComparison(I);
3294 return;
3295 }
3296
3297 assert(I.isUnsigned());
3298 if ((isa<Constant>(Val: I.getOperand(i_nocapture: 0)) || isa<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3299 handleRelationalComparisonExact(I);
3300 return;
3301 }
3302
3303 handleShadowOr(I);
3304 }
3305
3306 void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
3307
3308 void handleShift(BinaryOperator &I) {
3309 IRBuilder<> IRB(&I);
3310 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3311 // Otherwise perform the same shift on S1.
3312 Value *S1 = getShadow(I: &I, i: 0);
3313 Value *S2 = getShadow(I: &I, i: 1);
3314 Value *S2Conv =
3315 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3316 Value *V2 = I.getOperand(i_nocapture: 1);
3317 Value *Shift = IRB.CreateBinOp(Opc: I.getOpcode(), LHS: S1, RHS: V2);
3318 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3319 setOriginForNaryOp(I);
3320 }
3321
3322 void visitShl(BinaryOperator &I) { handleShift(I); }
3323 void visitAShr(BinaryOperator &I) { handleShift(I); }
3324 void visitLShr(BinaryOperator &I) { handleShift(I); }
3325
3326 void handleFunnelShift(IntrinsicInst &I) {
3327 IRBuilder<> IRB(&I);
3328 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3329 // Otherwise perform the same shift on S0 and S1.
3330 Value *S0 = getShadow(I: &I, i: 0);
3331 Value *S1 = getShadow(I: &I, i: 1);
3332 Value *S2 = getShadow(I: &I, i: 2);
3333 Value *S2Conv =
3334 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3335 Value *V2 = I.getOperand(i_nocapture: 2);
3336 Value *Shift = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: S2Conv->getType(),
3337 Args: {S0, S1, V2});
3338 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3339 setOriginForNaryOp(I);
3340 }
3341
3342 // Instrument bit manipulation intrinsics.
3343 // All of these intrinsics are Z = I(SRC, MASK)
3344 // where the types of all operands and the result match.
3345 // The following instrumentation happens to work for all of them:
3346 // Sz = I(Ssrc, MASK) | (sext (Smask != 0))
3347 void handleGenericBitManipulation(IntrinsicInst &I) {
3348 IRBuilder<> IRB(&I);
3349 Type *ShadowTy = getShadowTy(V: &I);
3350
3351 // If any bit of the mask operand is poisoned, then the whole thing is.
3352 Value *SMask = getShadow(I: &I, i: 1);
3353 SMask = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SMask, RHS: getCleanShadow(OrigTy: ShadowTy)),
3354 DestTy: ShadowTy);
3355 // Apply the same intrinsic to the shadow of the first operand.
3356 Value *S;
3357 if (Function *Func = I.getCalledFunction())
3358 S = IRB.CreateCall(Callee: Func, Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3359 else
3360 S = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: ShadowTy,
3361 Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3362
3363 setShadow(V: &I, SV: IRB.CreateOr(LHS: SMask, RHS: S));
3364 setOriginForNaryOp(I);
3365 }
3366
3367 /// Instrument llvm.memmove
3368 ///
3369 /// At this point we don't know if llvm.memmove will be inlined or not.
3370 /// If we don't instrument it and it gets inlined,
3371 /// our interceptor will not kick in and we will lose the memmove.
3372 /// If we instrument the call here, but it does not get inlined,
3373 /// we will memmove the shadow twice: which is bad in case
3374 /// of overlapping regions. So, we simply lower the intrinsic to a call.
3375 ///
3376 /// Similar situation exists for memcpy and memset.
3377 void visitMemMoveInst(MemMoveInst &I) {
3378 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3379 IRBuilder<> IRB(&I);
3380 IRB.CreateCall(Callee: MS.MemmoveFn,
3381 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3382 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3383 I.eraseFromParent();
3384 }
3385
3386 /// Instrument memcpy
3387 ///
3388 /// Similar to memmove: avoid copying shadow twice. This is somewhat
3389 /// unfortunate as it may slowdown small constant memcpys.
3390 /// FIXME: consider doing manual inline for small constant sizes and proper
3391 /// alignment.
3392 ///
3393 /// Note: This also handles memcpy.inline, which promises no calls to external
3394 /// functions as an optimization. However, with instrumentation enabled this
3395 /// is difficult to promise; additionally, we know that the MSan runtime
3396 /// exists and provides __msan_memcpy(). Therefore, we assume that with
3397 /// instrumentation it's safe to turn memcpy.inline into a call to
3398 /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
3399 /// itself, instrumentation should be disabled with the no_sanitize attribute.
3400 void visitMemCpyInst(MemCpyInst &I) {
3401 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3402 IRBuilder<> IRB(&I);
3403 IRB.CreateCall(Callee: MS.MemcpyFn,
3404 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3405 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3406 I.eraseFromParent();
3407 }
3408
3409 // Same as memcpy.
3410 void visitMemSetInst(MemSetInst &I) {
3411 IRBuilder<> IRB(&I);
3412 IRB.CreateCall(
3413 Callee: MS.MemsetFn,
3414 Args: {I.getArgOperand(i: 0),
3415 IRB.CreateIntCast(V: I.getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
3416 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3417 I.eraseFromParent();
3418 }
3419
3420 void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
3421
3422 void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
3423
3424 /// Handle vector store-like intrinsics.
3425 ///
3426 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
3427 /// has 1 pointer argument and 1 vector argument, returns void.
3428 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
3429 assert(I.arg_size() == 2);
3430
3431 IRBuilder<> IRB(&I);
3432 Value *Addr = I.getArgOperand(i: 0);
3433 Value *Shadow = getShadow(I: &I, i: 1);
3434 Value *ShadowPtr, *OriginPtr;
3435
3436 // We don't know the pointer alignment (could be unaligned SSE store!).
3437 // Have to assume to worst case.
3438 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
3439 Addr, IRB, ShadowTy: Shadow->getType(), Alignment: Align(1), /*isStore*/ true);
3440 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1));
3441
3442 if (ClCheckAccessAddress)
3443 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3444
3445 // FIXME: factor out common code from materializeStores
3446 if (MS.TrackOrigins)
3447 IRB.CreateStore(Val: getOrigin(I: &I, i: 1), Ptr: OriginPtr);
3448 return true;
3449 }
3450
3451 /// Handle vector load-like intrinsics.
3452 ///
3453 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
3454 /// has 1 pointer argument, returns a vector.
3455 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
3456 assert(I.arg_size() == 1);
3457
3458 IRBuilder<> IRB(&I);
3459 Value *Addr = I.getArgOperand(i: 0);
3460
3461 Type *ShadowTy = getShadowTy(V: &I);
3462 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
3463 if (PropagateShadow) {
3464 // We don't know the pointer alignment (could be unaligned SSE load!).
3465 // Have to assume to worst case.
3466 const Align Alignment = Align(1);
3467 std::tie(args&: ShadowPtr, args&: OriginPtr) =
3468 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3469 setShadow(V: &I,
3470 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
3471 } else {
3472 setShadow(V: &I, SV: getCleanShadow(V: &I));
3473 }
3474
3475 if (ClCheckAccessAddress)
3476 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3477
3478 if (MS.TrackOrigins) {
3479 if (PropagateShadow)
3480 setOrigin(V: &I, Origin: IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
3481 else
3482 setOrigin(V: &I, Origin: getCleanOrigin());
3483 }
3484 return true;
3485 }
3486
3487 /// Handle (SIMD arithmetic)-like intrinsics.
3488 ///
3489 /// Instrument intrinsics with any number of arguments of the same type [*],
3490 /// equal to the return type, plus a specified number of trailing flags of
3491 /// any type.
3492 ///
3493 /// [*] The type should be simple (no aggregates or pointers; vectors are
3494 /// fine).
3495 ///
3496 /// Caller guarantees that this intrinsic does not access memory.
3497 ///
3498 /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
3499 /// by this handler. See horizontalReduce().
3500 ///
3501 /// TODO: permutation intrinsics are also often incorrectly matched.
3502 [[maybe_unused]] bool
3503 maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
3504 unsigned int trailingFlags) {
3505 Type *RetTy = I.getType();
3506 if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy()))
3507 return false;
3508
3509 unsigned NumArgOperands = I.arg_size();
3510 assert(NumArgOperands >= trailingFlags);
3511 for (unsigned i = 0; i < NumArgOperands - trailingFlags; ++i) {
3512 Type *Ty = I.getArgOperand(i)->getType();
3513 if (Ty != RetTy)
3514 return false;
3515 }
3516
3517 IRBuilder<> IRB(&I);
3518 ShadowAndOriginCombiner SC(this, IRB);
3519 for (unsigned i = 0; i < NumArgOperands; ++i)
3520 SC.Add(V: I.getArgOperand(i));
3521 SC.Done(I: &I);
3522
3523 return true;
3524 }
3525
3526 /// Returns whether it was able to heuristically instrument unknown
3527 /// intrinsics.
3528 ///
3529 /// The main purpose of this code is to do something reasonable with all
3530 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
3531 /// We recognize several classes of intrinsics by their argument types and
3532 /// ModRefBehaviour and apply special instrumentation when we are reasonably
3533 /// sure that we know what the intrinsic does.
3534 ///
3535 /// We special-case intrinsics where this approach fails. See llvm.bswap
3536 /// handling as an example of that.
3537 bool maybeHandleUnknownIntrinsicUnlogged(IntrinsicInst &I) {
3538 unsigned NumArgOperands = I.arg_size();
3539 if (NumArgOperands == 0)
3540 return false;
3541
3542 if (NumArgOperands == 2 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3543 I.getArgOperand(i: 1)->getType()->isVectorTy() &&
3544 I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
3545 // This looks like a vector store.
3546 return handleVectorStoreIntrinsic(I);
3547 }
3548
3549 if (NumArgOperands == 1 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3550 I.getType()->isVectorTy() && I.onlyReadsMemory()) {
3551 // This looks like a vector load.
3552 return handleVectorLoadIntrinsic(I);
3553 }
3554
3555 if (I.doesNotAccessMemory())
3556 if (maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/0))
3557 return true;
3558
3559 // FIXME: detect and handle SSE maskstore/maskload?
3560 // Some cases are now handled in handleAVXMasked{Load,Store}.
3561 return false;
3562 }
3563
3564 bool maybeHandleUnknownIntrinsic(IntrinsicInst &I) {
3565 if (maybeHandleUnknownIntrinsicUnlogged(I)) {
3566 if (ClDumpHeuristicInstructions)
3567 dumpInst(I, Prefix: "Heuristic");
3568
3569 LLVM_DEBUG(dbgs() << "UNKNOWN INSTRUCTION HANDLED HEURISTICALLY: " << I
3570 << "\n");
3571 return true;
3572 } else
3573 return false;
3574 }
3575
3576 void handleInvariantGroup(IntrinsicInst &I) {
3577 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3578 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3579 }
3580
3581 void handleLifetimeStart(IntrinsicInst &I) {
3582 if (!PoisonStack)
3583 return;
3584 AllocaInst *AI = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
3585 if (AI)
3586 LifetimeStartList.push_back(Elt: std::make_pair(x: &I, y&: AI));
3587 }
3588
3589 void handleBswap(IntrinsicInst &I) {
3590 IRBuilder<> IRB(&I);
3591 Value *Op = I.getArgOperand(i: 0);
3592 Type *OpType = Op->getType();
3593 setShadow(V: &I, SV: IRB.CreateIntrinsic(ID: Intrinsic::bswap, OverloadTypes: ArrayRef(&OpType, 1),
3594 Args: getShadow(V: Op)));
3595 setOrigin(V: &I, Origin: getOrigin(V: Op));
3596 }
3597
3598 // Uninitialized bits are ok if they appear after the leading/trailing 0's
3599 // and a 1. If the input is all zero, it is fully initialized iff
3600 // !is_zero_poison.
3601 //
3602 // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
3603 // concrete value 0/1, and ? is an uninitialized bit:
3604 // - 0001 0??? is fully initialized
3605 // - 000? ???? is fully uninitialized (*)
3606 // - ???? ???? is fully uninitialized
3607 // - 0000 0000 is fully uninitialized if is_zero_poison,
3608 // fully initialized otherwise
3609 //
3610 // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
3611 // only need to poison 4 bits.
3612 //
3613 // OutputShadow =
3614 // ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
3615 // || (is_zero_poison && AllZeroSrc)
3616 void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
3617 IRBuilder<> IRB(&I);
3618 Value *Src = I.getArgOperand(i: 0);
3619 Value *SrcShadow = getShadow(V: Src);
3620
3621 Value *False = IRB.getInt1(V: false);
3622 Value *ConcreteZerosCount = IRB.CreateIntrinsic(
3623 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Src, /*is_zero_poison=*/False});
3624 Value *ShadowZerosCount = IRB.CreateIntrinsic(
3625 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {SrcShadow, /*is_zero_poison=*/False});
3626
3627 Value *CompareConcreteZeros = IRB.CreateICmpUGE(
3628 LHS: ConcreteZerosCount, RHS: ShadowZerosCount, Name: "_mscz_cmp_zeros");
3629
3630 Value *NotAllZeroShadow =
3631 IRB.CreateIsNotNull(Arg: SrcShadow, Name: "_mscz_shadow_not_null");
3632 Value *OutputShadow =
3633 IRB.CreateAnd(LHS: CompareConcreteZeros, RHS: NotAllZeroShadow, Name: "_mscz_main");
3634
3635 // If zero poison is requested, mix in with the shadow
3636 Constant *IsZeroPoison = cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3637 if (!IsZeroPoison->isNullValue()) {
3638 Value *BoolZeroPoison = IRB.CreateIsNull(Arg: Src, Name: "_mscz_bzp");
3639 OutputShadow = IRB.CreateOr(LHS: OutputShadow, RHS: BoolZeroPoison, Name: "_mscz_bs");
3640 }
3641
3642 OutputShadow = IRB.CreateSExt(V: OutputShadow, DestTy: getShadowTy(V: Src), Name: "_mscz_os");
3643
3644 setShadow(V: &I, SV: OutputShadow);
3645 setOriginForNaryOp(I);
3646 }
3647
3648 /// Some instructions have additional zero-elements in the return type
3649 /// e.g., <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, ...)
3650 ///
3651 /// This function will return a vector type with the same number of elements
3652 /// as the input, but same per-element width as the return value e.g.,
3653 /// <8 x i8>.
3654 FixedVectorType *maybeShrinkVectorShadowType(Value *Src, IntrinsicInst &I) {
3655 assert(isa<FixedVectorType>(getShadowTy(&I)));
3656 FixedVectorType *ShadowType = cast<FixedVectorType>(Val: getShadowTy(V: &I));
3657
3658 // TODO: generalize beyond 2x?
3659 if (ShadowType->getElementCount() ==
3660 cast<VectorType>(Val: Src->getType())->getElementCount() * 2)
3661 ShadowType = FixedVectorType::getHalfElementsVectorType(VTy: ShadowType);
3662
3663 assert(ShadowType->getElementCount() ==
3664 cast<VectorType>(Src->getType())->getElementCount());
3665
3666 return ShadowType;
3667 }
3668
3669 /// Doubles the length of a vector shadow (extending with zeros) if necessary
3670 /// to match the length of the shadow for the instruction.
3671 /// If scalar types of the vectors are different, it will use the type of the
3672 /// input vector.
3673 /// This is more type-safe than CreateShadowCast().
3674 Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
3675 IRBuilder<> IRB(&I);
3676 assert(isa<FixedVectorType>(Shadow->getType()));
3677 assert(isa<FixedVectorType>(I.getType()));
3678
3679 Value *FullShadow = getCleanShadow(V: &I);
3680 unsigned ShadowNumElems =
3681 cast<FixedVectorType>(Val: Shadow->getType())->getNumElements();
3682 unsigned FullShadowNumElems =
3683 cast<FixedVectorType>(Val: FullShadow->getType())->getNumElements();
3684
3685 assert((ShadowNumElems == FullShadowNumElems) ||
3686 (ShadowNumElems * 2 == FullShadowNumElems));
3687
3688 if (ShadowNumElems == FullShadowNumElems) {
3689 FullShadow = Shadow;
3690 } else {
3691 // TODO: generalize beyond 2x?
3692 SmallVector<int, 32> ShadowMask(FullShadowNumElems);
3693 std::iota(first: ShadowMask.begin(), last: ShadowMask.end(), value: 0);
3694
3695 // Append zeros
3696 FullShadow =
3697 IRB.CreateShuffleVector(V1: Shadow, V2: getCleanShadow(V: Shadow), Mask: ShadowMask);
3698 }
3699
3700 return FullShadow;
3701 }
3702
3703 /// Handle x86 SSE vector conversion.
3704 ///
3705 /// e.g., single-precision to half-precision conversion:
3706 /// <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
3707 /// <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0)
3708 ///
3709 /// floating-point to integer:
3710 /// <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>)
3711 /// <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>)
3712 ///
3713 /// Note: if the output has more elements, they are zero-initialized (and
3714 /// therefore the shadow will also be initialized).
3715 ///
3716 /// This differs from handleSSEVectorConvertIntrinsic() because it
3717 /// propagates uninitialized shadow (instead of checking the shadow).
3718 void handleSSEVectorConvertIntrinsicByProp(IntrinsicInst &I,
3719 bool HasRoundingMode) {
3720 if (HasRoundingMode) {
3721 assert(I.arg_size() == 2);
3722 [[maybe_unused]] Value *RoundingMode = I.getArgOperand(i: 1);
3723 assert(RoundingMode->getType()->isIntegerTy());
3724 } else {
3725 assert(I.arg_size() == 1);
3726 }
3727
3728 Value *Src = I.getArgOperand(i: 0);
3729 assert(Src->getType()->isVectorTy());
3730
3731 // The return type might have more elements than the input.
3732 // Temporarily shrink the return type's number of elements.
3733 VectorType *ShadowType = maybeShrinkVectorShadowType(Src, I);
3734
3735 IRBuilder<> IRB(&I);
3736 Value *S0 = getShadow(I: &I, i: 0);
3737
3738 /// For scalars:
3739 /// Since they are converting to and/or from floating-point, the output is:
3740 /// - fully uninitialized if *any* bit of the input is uninitialized
3741 /// - fully ininitialized if all bits of the input are ininitialized
3742 /// We apply the same principle on a per-field basis for vectors.
3743 Value *Shadow =
3744 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)), DestTy: ShadowType);
3745
3746 // The return type might have more elements than the input.
3747 // Extend the return type back to its original width if necessary.
3748 Value *FullShadow = maybeExtendVectorShadowWithZeros(Shadow, I);
3749
3750 setShadow(V: &I, SV: FullShadow);
3751 setOriginForNaryOp(I);
3752 }
3753
3754 // Instrument x86 SSE vector convert intrinsic.
3755 //
3756 // This function instruments intrinsics like cvtsi2ss:
3757 // %Out = int_xxx_cvtyyy(%ConvertOp)
3758 // or
3759 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
3760 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
3761 // number \p Out elements, and (if has 2 arguments) copies the rest of the
3762 // elements from \p CopyOp.
3763 // In most cases conversion involves floating-point value which may trigger a
3764 // hardware exception when not fully initialized. For this reason we require
3765 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
3766 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
3767 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
3768 // return a fully initialized value.
3769 //
3770 // For Arm NEON vector convert intrinsics, see
3771 // handleNEONVectorConvertIntrinsic().
3772 void handleSSEVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
3773 bool HasRoundingMode = false) {
3774 IRBuilder<> IRB(&I);
3775 Value *CopyOp, *ConvertOp;
3776
3777 assert((!HasRoundingMode ||
3778 isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
3779 "Invalid rounding mode");
3780
3781 switch (I.arg_size() - HasRoundingMode) {
3782 case 2:
3783 CopyOp = I.getArgOperand(i: 0);
3784 ConvertOp = I.getArgOperand(i: 1);
3785 break;
3786 case 1:
3787 ConvertOp = I.getArgOperand(i: 0);
3788 CopyOp = nullptr;
3789 break;
3790 default:
3791 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
3792 }
3793
3794 // The first *NumUsedElements* elements of ConvertOp are converted to the
3795 // same number of output elements. The rest of the output is copied from
3796 // CopyOp, or (if not available) filled with zeroes.
3797 // Combine shadow for elements of ConvertOp that are used in this operation,
3798 // and insert a check.
3799 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
3800 // int->any conversion.
3801 Value *ConvertShadow = getShadow(V: ConvertOp);
3802 Value *AggShadow = nullptr;
3803 if (ConvertOp->getType()->isVectorTy()) {
3804 AggShadow = IRB.CreateExtractElement(
3805 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
3806 for (int i = 1; i < NumUsedElements; ++i) {
3807 Value *MoreShadow = IRB.CreateExtractElement(
3808 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3809 AggShadow = IRB.CreateOr(LHS: AggShadow, RHS: MoreShadow);
3810 }
3811 } else {
3812 AggShadow = ConvertShadow;
3813 }
3814 assert(AggShadow->getType()->isIntegerTy());
3815 insertCheckShadow(Shadow: AggShadow, Origin: getOrigin(V: ConvertOp), OrigIns: &I);
3816
3817 // Build result shadow by zero-filling parts of CopyOp shadow that come from
3818 // ConvertOp.
3819 if (CopyOp) {
3820 assert(CopyOp->getType() == I.getType());
3821 assert(CopyOp->getType()->isVectorTy());
3822 Value *ResultShadow = getShadow(V: CopyOp);
3823 Type *EltTy = cast<VectorType>(Val: ResultShadow->getType())->getElementType();
3824 for (int i = 0; i < NumUsedElements; ++i) {
3825 ResultShadow = IRB.CreateInsertElement(
3826 Vec: ResultShadow, NewElt: ConstantInt::getNullValue(Ty: EltTy),
3827 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3828 }
3829 setShadow(V: &I, SV: ResultShadow);
3830 setOrigin(V: &I, Origin: getOrigin(V: CopyOp));
3831 } else {
3832 setShadow(V: &I, SV: getCleanShadow(V: &I));
3833 setOrigin(V: &I, Origin: getCleanOrigin());
3834 }
3835 }
3836
3837 // Given a scalar or vector, extract lower 64 bits (or less), and return all
3838 // zeroes if it is zero, and all ones otherwise.
3839 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3840 if (S->getType()->isVectorTy())
3841 S = CreateShadowCast(IRB, V: S, dstTy: IRB.getInt64Ty(), /* Signed */ true);
3842 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
3843 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3844 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3845 }
3846
3847 // Given a vector, extract its first element, and return all
3848 // zeroes if it is zero, and all ones otherwise.
3849 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3850 Value *S1 = IRB.CreateExtractElement(Vec: S, Idx: (uint64_t)0);
3851 Value *S2 = IRB.CreateICmpNE(LHS: S1, RHS: getCleanShadow(V: S1));
3852 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3853 }
3854
3855 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3856 Type *T = S->getType();
3857 assert(T->isVectorTy());
3858 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3859 return IRB.CreateSExt(V: S2, DestTy: T);
3860 }
3861
3862 // Instrument vector shift intrinsic.
3863 //
3864 // This function instruments intrinsics like int_x86_avx2_psll_w.
3865 // Intrinsic shifts %In by %ShiftSize bits.
3866 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
3867 // size, and the rest is ignored. Behavior is defined even if shift size is
3868 // greater than register (or field) width.
3869 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3870 assert(I.arg_size() == 2);
3871 IRBuilder<> IRB(&I);
3872 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3873 // Otherwise perform the same shift on S1.
3874 Value *S1 = getShadow(I: &I, i: 0);
3875 Value *S2 = getShadow(I: &I, i: 1);
3876 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S: S2)
3877 : Lower64ShadowExtend(IRB, S: S2, T: getShadowTy(V: &I));
3878 Value *V1 = I.getOperand(i_nocapture: 0);
3879 Value *V2 = I.getOperand(i_nocapture: 1);
3880 Value *Shift = IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
3881 Args: {IRB.CreateBitCast(V: S1, DestTy: V1->getType()), V2});
3882 Shift = IRB.CreateBitCast(V: Shift, DestTy: getShadowTy(V: &I));
3883 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3884 setOriginForNaryOp(I);
3885 }
3886
3887 // Get an MMX-sized (64-bit) vector type, or optionally, other sized
3888 // vectors.
3889 Type *getMMXVectorTy(unsigned EltSizeInBits,
3890 unsigned X86_MMXSizeInBits = 64) {
3891 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3892 "Illegal MMX vector element size");
3893 return FixedVectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSizeInBits),
3894 NumElts: X86_MMXSizeInBits / EltSizeInBits);
3895 }
3896
3897 // Returns a signed counterpart for an (un)signed-saturate-and-pack
3898 // intrinsic.
3899 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
3900 switch (id) {
3901 case Intrinsic::x86_sse2_packsswb_128:
3902 case Intrinsic::x86_sse2_packuswb_128:
3903 return Intrinsic::x86_sse2_packsswb_128;
3904
3905 case Intrinsic::x86_sse2_packssdw_128:
3906 case Intrinsic::x86_sse41_packusdw:
3907 return Intrinsic::x86_sse2_packssdw_128;
3908
3909 case Intrinsic::x86_avx2_packsswb:
3910 case Intrinsic::x86_avx2_packuswb:
3911 return Intrinsic::x86_avx2_packsswb;
3912
3913 case Intrinsic::x86_avx2_packssdw:
3914 case Intrinsic::x86_avx2_packusdw:
3915 return Intrinsic::x86_avx2_packssdw;
3916
3917 case Intrinsic::x86_mmx_packsswb:
3918 case Intrinsic::x86_mmx_packuswb:
3919 return Intrinsic::x86_mmx_packsswb;
3920
3921 case Intrinsic::x86_mmx_packssdw:
3922 return Intrinsic::x86_mmx_packssdw;
3923
3924 case Intrinsic::x86_avx512_packssdw_512:
3925 case Intrinsic::x86_avx512_packusdw_512:
3926 return Intrinsic::x86_avx512_packssdw_512;
3927
3928 case Intrinsic::x86_avx512_packsswb_512:
3929 case Intrinsic::x86_avx512_packuswb_512:
3930 return Intrinsic::x86_avx512_packsswb_512;
3931
3932 default:
3933 llvm_unreachable("unexpected intrinsic id");
3934 }
3935 }
3936
3937 // Instrument vector pack intrinsic.
3938 //
3939 // This function instruments intrinsics like x86_mmx_packsswb, that
3940 // packs elements of 2 input vectors into half as many bits with saturation.
3941 // Shadow is propagated with the signed variant of the same intrinsic applied
3942 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3943 // MMXEltSizeInBits is used only for x86mmx arguments.
3944 //
3945 // TODO: consider using GetMinMaxUnsigned() to handle saturation precisely
3946 void handleVectorPackIntrinsic(IntrinsicInst &I,
3947 unsigned MMXEltSizeInBits = 0) {
3948 assert(I.arg_size() == 2);
3949 IRBuilder<> IRB(&I);
3950 Value *S1 = getShadow(I: &I, i: 0);
3951 Value *S2 = getShadow(I: &I, i: 1);
3952 assert(S1->getType()->isVectorTy());
3953
3954 // SExt and ICmpNE below must apply to individual elements of input vectors.
3955 // In case of x86mmx arguments, cast them to appropriate vector types and
3956 // back.
3957 Type *T =
3958 MMXEltSizeInBits ? getMMXVectorTy(EltSizeInBits: MMXEltSizeInBits) : S1->getType();
3959 if (MMXEltSizeInBits) {
3960 S1 = IRB.CreateBitCast(V: S1, DestTy: T);
3961 S2 = IRB.CreateBitCast(V: S2, DestTy: T);
3962 }
3963 Value *S1_ext =
3964 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S1, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3965 Value *S2_ext =
3966 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3967 if (MMXEltSizeInBits) {
3968 S1_ext = IRB.CreateBitCast(V: S1_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3969 S2_ext = IRB.CreateBitCast(V: S2_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3970 }
3971
3972 Value *S = IRB.CreateIntrinsic(ID: getSignedPackIntrinsic(id: I.getIntrinsicID()),
3973 Args: {S1_ext, S2_ext}, /*FMFSource=*/nullptr,
3974 Name: "_msprop_vector_pack");
3975 if (MMXEltSizeInBits)
3976 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3977 setShadow(V: &I, SV: S);
3978 setOriginForNaryOp(I);
3979 }
3980
3981 // Convert `Mask` into `<n x i1>`.
3982 Constant *createDppMask(unsigned Width, unsigned Mask) {
3983 SmallVector<Constant *, 4> R(Width);
3984 for (auto &M : R) {
3985 M = ConstantInt::getBool(Context&: F.getContext(), V: Mask & 1);
3986 Mask >>= 1;
3987 }
3988 return ConstantVector::get(V: R);
3989 }
3990
3991 // Calculate output shadow as array of booleans `<n x i1>`, assuming if any
3992 // arg is poisoned, entire dot product is poisoned.
3993 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
3994 unsigned DstMask) {
3995 const unsigned Width =
3996 cast<FixedVectorType>(Val: S->getType())->getNumElements();
3997
3998 S = IRB.CreateSelect(C: createDppMask(Width, Mask: SrcMask), True: S,
3999 False: Constant::getNullValue(Ty: S->getType()));
4000 Value *SElem = IRB.CreateOrReduce(Src: S);
4001 Value *IsClean = IRB.CreateIsNull(Arg: SElem, Name: "_msdpp");
4002 Value *DstMaskV = createDppMask(Width, Mask: DstMask);
4003
4004 return IRB.CreateSelect(
4005 C: IsClean, True: Constant::getNullValue(Ty: DstMaskV->getType()), False: DstMaskV);
4006 }
4007
4008 // See `Intel Intrinsics Guide` for `_dp_p*` instructions.
4009 //
4010 // 2 and 4 element versions produce single scalar of dot product, and then
4011 // puts it into elements of output vector, selected by 4 lowest bits of the
4012 // mask. Top 4 bits of the mask control which elements of input to use for dot
4013 // product.
4014 //
4015 // 8 element version mask still has only 4 bit for input, and 4 bit for output
4016 // mask. According to the spec it just operates as 4 element version on first
4017 // 4 elements of inputs and output, and then on last 4 elements of inputs and
4018 // output.
4019 void handleDppIntrinsic(IntrinsicInst &I) {
4020 IRBuilder<> IRB(&I);
4021
4022 Value *S0 = getShadow(I: &I, i: 0);
4023 Value *S1 = getShadow(I: &I, i: 1);
4024 Value *S = IRB.CreateOr(LHS: S0, RHS: S1);
4025
4026 const unsigned Width =
4027 cast<FixedVectorType>(Val: S->getType())->getNumElements();
4028 assert(Width == 2 || Width == 4 || Width == 8);
4029
4030 const unsigned Mask = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4031 const unsigned SrcMask = Mask >> 4;
4032 const unsigned DstMask = Mask & 0xf;
4033
4034 // Calculate shadow as `<n x i1>`.
4035 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
4036 if (Width == 8) {
4037 // First 4 elements of shadow are already calculated. `makeDppShadow`
4038 // operats on 32 bit masks, so we can just shift masks, and repeat.
4039 SI1 = IRB.CreateOr(
4040 LHS: SI1, RHS: findDppPoisonedOutput(IRB, S, SrcMask: SrcMask << 4, DstMask: DstMask << 4));
4041 }
4042 // Extend to real size of shadow, poisoning either all or none bits of an
4043 // element.
4044 S = IRB.CreateSExt(V: SI1, DestTy: S->getType(), Name: "_msdpp");
4045
4046 setShadow(V: &I, SV: S);
4047 setOriginForNaryOp(I);
4048 }
4049
4050 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
4051 C = CreateAppToShadowCast(IRB, V: C);
4052 FixedVectorType *FVT = cast<FixedVectorType>(Val: C->getType());
4053 unsigned ElSize = FVT->getElementType()->getPrimitiveSizeInBits();
4054 C = IRB.CreateAShr(LHS: C, RHS: ElSize - 1);
4055 FVT = FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: FVT->getNumElements());
4056 return IRB.CreateTrunc(V: C, DestTy: FVT);
4057 }
4058
4059 // `blendv(f, t, c)` is effectively `select(c[top_bit], t, f)`.
4060 void handleBlendvIntrinsic(IntrinsicInst &I) {
4061 Value *C = I.getOperand(i_nocapture: 2);
4062 Value *T = I.getOperand(i_nocapture: 1);
4063 Value *F = I.getOperand(i_nocapture: 0);
4064
4065 Value *Sc = getShadow(I: &I, i: 2);
4066 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
4067
4068 {
4069 IRBuilder<> IRB(&I);
4070 // Extract top bit from condition and its shadow.
4071 C = convertBlendvToSelectMask(IRB, C);
4072 Sc = convertBlendvToSelectMask(IRB, C: Sc);
4073
4074 setShadow(V: C, SV: Sc);
4075 setOrigin(V: C, Origin: Oc);
4076 }
4077
4078 handleSelectLikeInst(I, B: C, C: T, D: F);
4079 }
4080
4081 // Instrument sum-of-absolute-differences intrinsic.
4082 void handleVectorSadIntrinsic(IntrinsicInst &I, bool IsMMX = false) {
4083 const unsigned SignificantBitsPerResultElement = 16;
4084 Type *ResTy = IsMMX ? IntegerType::get(C&: *MS.C, NumBits: 64) : I.getType();
4085 unsigned ZeroBitsPerResultElement =
4086 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
4087
4088 IRBuilder<> IRB(&I);
4089 auto *Shadow0 = getShadow(I: &I, i: 0);
4090 auto *Shadow1 = getShadow(I: &I, i: 1);
4091 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4092 S = IRB.CreateBitCast(V: S, DestTy: ResTy);
4093 S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
4094 DestTy: ResTy);
4095 S = IRB.CreateLShr(LHS: S, RHS: ZeroBitsPerResultElement);
4096 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
4097 setShadow(V: &I, SV: S);
4098 setOriginForNaryOp(I);
4099 }
4100
4101 // Instrument dot-product / multiply-add(-accumulate)? intrinsics.
4102 //
4103 // e.g., Two operands:
4104 // <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> %b)
4105 //
4106 // Two operands which require an EltSizeInBits override:
4107 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %a, <1 x i64> %b)
4108 //
4109 // Three operands:
4110 // <4 x i32> @llvm.x86.avx512.vpdpbusd.128
4111 // (<4 x i32> %s, <16 x i8> %a, <16 x i8> %b)
4112 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
4113 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
4114 // (these are equivalent to multiply-add on %a and %b, followed by
4115 // adding/"accumulating" %s. "Accumulation" stores the result in one
4116 // of the source registers, but this accumulate vs. add distinction
4117 // is lost when dealing with LLVM intrinsics.)
4118 //
4119 // ZeroPurifies means that multiplying a known-zero with an uninitialized
4120 // value results in an initialized value. This is applicable for integer
4121 // multiplication, but not floating-point (counter-example: NaN).
4122 void handleVectorDotProductIntrinsic(IntrinsicInst &I,
4123 unsigned ReductionFactor,
4124 bool ZeroPurifies,
4125 unsigned EltSizeInBits,
4126 enum OddOrEvenLanes Lanes) {
4127 IRBuilder<> IRB(&I);
4128
4129 [[maybe_unused]] FixedVectorType *ReturnType =
4130 cast<FixedVectorType>(Val: I.getType());
4131 assert(isa<FixedVectorType>(ReturnType));
4132
4133 // Vectors A and B, and shadows
4134 Value *Va = nullptr;
4135 Value *Vb = nullptr;
4136 Value *Sa = nullptr;
4137 Value *Sb = nullptr;
4138
4139 assert(I.arg_size() == 2 || I.arg_size() == 3);
4140 if (I.arg_size() == 2) {
4141 assert(Lanes == kBothLanes);
4142
4143 Va = I.getOperand(i_nocapture: 0);
4144 Vb = I.getOperand(i_nocapture: 1);
4145
4146 Sa = getShadow(I: &I, i: 0);
4147 Sb = getShadow(I: &I, i: 1);
4148 } else if (I.arg_size() == 3) {
4149 // Operand 0 is the accumulator. We will deal with that below.
4150 Va = I.getOperand(i_nocapture: 1);
4151 Vb = I.getOperand(i_nocapture: 2);
4152
4153 Sa = getShadow(I: &I, i: 1);
4154 Sb = getShadow(I: &I, i: 2);
4155
4156 if (Lanes == kEvenLanes || Lanes == kOddLanes) {
4157 // Convert < S0, S1, S2, S3, S4, S5, S6, S7 >
4158 // to < S0, S0, S2, S2, S4, S4, S6, S6 > (if even)
4159 // to < S1, S1, S3, S3, S5, S5, S7, S7 > (if odd)
4160 //
4161 // Note: for aarch64.neon.bfmlalb/t, the odd/even-indexed values are
4162 // zeroed, not duplicated. However, for shadow propagation, this
4163 // distinction is unimportant because Step 1 below will squeeze
4164 // each pair of elements (e.g., [S0, S0]) into a single bit, and
4165 // we only care if it is fully initialized.
4166
4167 FixedVectorType *InputShadowType = cast<FixedVectorType>(Val: Sa->getType());
4168 unsigned Width = InputShadowType->getNumElements();
4169
4170 Sa = IRB.CreateShuffleVector(
4171 V: Sa, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4172 Sb = IRB.CreateShuffleVector(
4173 V: Sb, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4174 }
4175 }
4176
4177 FixedVectorType *ParamType = cast<FixedVectorType>(Val: Va->getType());
4178 assert(ParamType == Vb->getType());
4179
4180 assert(ParamType->getPrimitiveSizeInBits() ==
4181 ReturnType->getPrimitiveSizeInBits());
4182
4183 if (I.arg_size() == 3) {
4184 [[maybe_unused]] auto *AccumulatorType =
4185 cast<FixedVectorType>(Val: I.getOperand(i_nocapture: 0)->getType());
4186 assert(AccumulatorType == ReturnType);
4187 }
4188
4189 FixedVectorType *ImplicitReturnType =
4190 cast<FixedVectorType>(Val: getShadowTy(OrigTy: ReturnType));
4191 // Step 1: instrument multiplication of corresponding vector elements
4192 if (EltSizeInBits) {
4193 ImplicitReturnType = cast<FixedVectorType>(
4194 Val: getMMXVectorTy(EltSizeInBits: EltSizeInBits * ReductionFactor,
4195 X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4196 ParamType = cast<FixedVectorType>(
4197 Val: getMMXVectorTy(EltSizeInBits, X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4198
4199 Va = IRB.CreateBitCast(V: Va, DestTy: ParamType);
4200 Vb = IRB.CreateBitCast(V: Vb, DestTy: ParamType);
4201
4202 Sa = IRB.CreateBitCast(V: Sa, DestTy: getShadowTy(OrigTy: ParamType));
4203 Sb = IRB.CreateBitCast(V: Sb, DestTy: getShadowTy(OrigTy: ParamType));
4204 } else {
4205 assert(ParamType->getNumElements() ==
4206 ReturnType->getNumElements() * ReductionFactor);
4207 }
4208
4209 // Each element of the vector is represented by a single bit (poisoned or
4210 // not) e.g., <8 x i1>.
4211 Value *SaNonZero = IRB.CreateIsNotNull(Arg: Sa);
4212 Value *SbNonZero = IRB.CreateIsNotNull(Arg: Sb);
4213 Value *And;
4214 if (ZeroPurifies) {
4215 // Multiplying an *initialized* zero by an uninitialized element results
4216 // in an initialized zero element.
4217 //
4218 // This is analogous to bitwise AND, where "AND" of 0 and a poisoned value
4219 // results in an unpoisoned value.
4220 Value *VaInt = Va;
4221 Value *VbInt = Vb;
4222 if (!Va->getType()->isIntegerTy()) {
4223 VaInt = CreateAppToShadowCast(IRB, V: Va);
4224 VbInt = CreateAppToShadowCast(IRB, V: Vb);
4225 }
4226
4227 // We check for non-zero on a per-element basis, not per-bit.
4228 Value *VaNonZero = IRB.CreateIsNotNull(Arg: VaInt);
4229 Value *VbNonZero = IRB.CreateIsNotNull(Arg: VbInt);
4230
4231 And = handleBitwiseAnd(IRB, V1: VaNonZero, V2: VbNonZero, S1: SaNonZero, S2: SbNonZero);
4232 } else {
4233 And = IRB.CreateOr(Ops: {SaNonZero, SbNonZero});
4234 }
4235
4236 // Extend <8 x i1> to <8 x i16>.
4237 // (The real pmadd intrinsic would have computed intermediate values of
4238 // <8 x i32>, but that is irrelevant for our shadow purposes because we
4239 // consider each element to be either fully initialized or fully
4240 // uninitialized.)
4241 And = IRB.CreateSExt(V: And, DestTy: Sa->getType());
4242
4243 // Step 2: instrument horizontal add
4244 // We don't need bit-precise horizontalReduce because we only want to check
4245 // if each pair/quad of elements is fully zero.
4246 // Cast to <4 x i32>.
4247 Value *Horizontal = IRB.CreateBitCast(V: And, DestTy: ImplicitReturnType);
4248
4249 // Compute <4 x i1>, then extend back to <4 x i32>.
4250 Value *OutShadow = IRB.CreateSExt(
4251 V: IRB.CreateICmpNE(LHS: Horizontal,
4252 RHS: Constant::getNullValue(Ty: Horizontal->getType())),
4253 DestTy: ImplicitReturnType);
4254
4255 // Cast it back to the required fake return type (if MMX: <1 x i64>; for
4256 // AVX, it is already correct).
4257 if (EltSizeInBits)
4258 OutShadow = CreateShadowCast(IRB, V: OutShadow, dstTy: getShadowTy(V: &I));
4259
4260 // Step 3 (if applicable): instrument accumulator
4261 if (I.arg_size() == 3)
4262 OutShadow = IRB.CreateOr(LHS: OutShadow, RHS: getShadow(I: &I, i: 0));
4263
4264 setShadow(V: &I, SV: OutShadow);
4265 setOriginForNaryOp(I);
4266 }
4267
4268 // Instrument compare-packed intrinsic.
4269 //
4270 // x86 has the predicate as the third operand, which is ImmArg e.g.,
4271 // - <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8)
4272 // - <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8)
4273 //
4274 // while Arm has separate intrinsics for >= and > e.g.,
4275 // - <2 x i32> @llvm.aarch64.neon.facge.v2i32.v2f32
4276 // (<2 x float> %A, <2 x float>)
4277 // - <2 x i32> @llvm.aarch64.neon.facgt.v2i32.v2f32
4278 // (<2 x float> %A, <2 x float>)
4279 //
4280 // Bonus: this also handles scalar cases e.g.,
4281 // - i32 @llvm.aarch64.neon.facgt.i32.f32(float %A, float %B)
4282 void handleVectorComparePackedIntrinsic(IntrinsicInst &I,
4283 bool PredicateAsOperand) {
4284 if (PredicateAsOperand) {
4285 assert(I.arg_size() == 3);
4286 assert(I.paramHasAttr(2, Attribute::ImmArg));
4287 } else
4288 assert(I.arg_size() == 2);
4289
4290 IRBuilder<> IRB(&I);
4291
4292 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
4293 // all-ones shadow.
4294 Type *ResTy = getShadowTy(V: &I);
4295 auto *Shadow0 = getShadow(I: &I, i: 0);
4296 auto *Shadow1 = getShadow(I: &I, i: 1);
4297 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4298 Value *S = IRB.CreateSExt(
4299 V: IRB.CreateICmpNE(LHS: S0, RHS: Constant::getNullValue(Ty: ResTy)), DestTy: ResTy);
4300 setShadow(V: &I, SV: S);
4301 setOriginForNaryOp(I);
4302 }
4303
4304 // Instrument compare-scalar intrinsic.
4305 // This handles both cmp* intrinsics which return the result in the first
4306 // element of a vector, and comi* which return the result as i32.
4307 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
4308 IRBuilder<> IRB(&I);
4309 auto *Shadow0 = getShadow(I: &I, i: 0);
4310 auto *Shadow1 = getShadow(I: &I, i: 1);
4311 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4312 Value *S = LowerElementShadowExtend(IRB, S: S0, T: getShadowTy(V: &I));
4313 setShadow(V: &I, SV: S);
4314 setOriginForNaryOp(I);
4315 }
4316
4317 // Instrument generic vector reduction intrinsics
4318 // by ORing together all their fields.
4319 //
4320 // If AllowShadowCast is true, the return type does not need to be the same
4321 // type as the fields
4322 // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
4323 void handleVectorReduceIntrinsic(IntrinsicInst &I, bool AllowShadowCast) {
4324 assert(I.arg_size() == 1);
4325
4326 IRBuilder<> IRB(&I);
4327 Value *S = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 0));
4328 if (AllowShadowCast)
4329 S = CreateShadowCast(IRB, V: S, dstTy: getShadowTy(V: &I));
4330 else
4331 assert(S->getType() == getShadowTy(&I));
4332 setShadow(V: &I, SV: S);
4333 setOriginForNaryOp(I);
4334 }
4335
4336 // Similar to handleVectorReduceIntrinsic but with an initial starting value.
4337 // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
4338 // %a1)
4339 // shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
4340 //
4341 // The type of the return value, initial starting value, and elements of the
4342 // vector must be identical.
4343 void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
4344 assert(I.arg_size() == 2);
4345
4346 IRBuilder<> IRB(&I);
4347 Value *Shadow0 = getShadow(I: &I, i: 0);
4348 Value *Shadow1 = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 1));
4349 assert(Shadow0->getType() == Shadow1->getType());
4350 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4351 assert(S->getType() == getShadowTy(&I));
4352 setShadow(V: &I, SV: S);
4353 setOriginForNaryOp(I);
4354 }
4355
4356 // Instrument vector.reduce.or intrinsic.
4357 // Valid (non-poisoned) set bits in the operand pull low the
4358 // corresponding shadow bits.
4359 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
4360 assert(I.arg_size() == 1);
4361
4362 IRBuilder<> IRB(&I);
4363 Value *OperandShadow = getShadow(I: &I, i: 0);
4364 Value *OperandUnsetBits = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
4365 Value *OperandUnsetOrPoison = IRB.CreateOr(LHS: OperandUnsetBits, RHS: OperandShadow);
4366 // Bit N is clean if any field's bit N is 1 and unpoison
4367 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandUnsetOrPoison);
4368 // Otherwise, it is clean if every field's bit N is unpoison
4369 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4370 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4371
4372 setShadow(V: &I, SV: S);
4373 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4374 }
4375
4376 // Instrument vector.reduce.and intrinsic.
4377 // Valid (non-poisoned) unset bits in the operand pull down the
4378 // corresponding shadow bits.
4379 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
4380 assert(I.arg_size() == 1);
4381
4382 IRBuilder<> IRB(&I);
4383 Value *OperandShadow = getShadow(I: &I, i: 0);
4384 Value *OperandSetOrPoison = IRB.CreateOr(LHS: I.getOperand(i_nocapture: 0), RHS: OperandShadow);
4385 // Bit N is clean if any field's bit N is 0 and unpoison
4386 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandSetOrPoison);
4387 // Otherwise, it is clean if every field's bit N is unpoison
4388 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4389 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4390
4391 setShadow(V: &I, SV: S);
4392 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4393 }
4394
4395 void handleStmxcsr(IntrinsicInst &I) {
4396 IRBuilder<> IRB(&I);
4397 Value *Addr = I.getArgOperand(i: 0);
4398 Type *Ty = IRB.getInt32Ty();
4399 Value *ShadowPtr =
4400 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment: Align(1), /*isStore*/ true).first;
4401
4402 IRB.CreateStore(Val: getCleanShadow(OrigTy: Ty), Ptr: ShadowPtr);
4403
4404 if (ClCheckAccessAddress)
4405 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4406 }
4407
4408 void handleLdmxcsr(IntrinsicInst &I) {
4409 if (!InsertChecks)
4410 return;
4411
4412 IRBuilder<> IRB(&I);
4413 Value *Addr = I.getArgOperand(i: 0);
4414 Type *Ty = IRB.getInt32Ty();
4415 const Align Alignment = Align(1);
4416 Value *ShadowPtr, *OriginPtr;
4417 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4418 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment, /*isStore*/ false);
4419
4420 if (ClCheckAccessAddress)
4421 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4422
4423 Value *Shadow = IRB.CreateAlignedLoad(Ty, Ptr: ShadowPtr, Align: Alignment, Name: "_ldmxcsr");
4424 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr)
4425 : getCleanOrigin();
4426 insertCheckShadow(Shadow, Origin, OrigIns: &I);
4427 }
4428
4429 void handleMaskedExpandLoad(IntrinsicInst &I) {
4430 IRBuilder<> IRB(&I);
4431 Value *Ptr = I.getArgOperand(i: 0);
4432 MaybeAlign Align = I.getParamAlign(ArgNo: 0);
4433 Value *Mask = I.getArgOperand(i: 1);
4434 Value *PassThru = I.getArgOperand(i: 2);
4435
4436 if (ClCheckAccessAddress) {
4437 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4438 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4439 }
4440
4441 if (!PropagateShadow) {
4442 setShadow(V: &I, SV: getCleanShadow(V: &I));
4443 setOrigin(V: &I, Origin: getCleanOrigin());
4444 return;
4445 }
4446
4447 Type *ShadowTy = getShadowTy(V: &I);
4448 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4449 auto [ShadowPtr, OriginPtr] =
4450 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ false);
4451
4452 Value *Shadow =
4453 IRB.CreateMaskedExpandLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align, Mask,
4454 PassThru: getShadow(V: PassThru), Name: "_msmaskedexpload");
4455
4456 setShadow(V: &I, SV: Shadow);
4457
4458 // TODO: Store origins.
4459 setOrigin(V: &I, Origin: getCleanOrigin());
4460 }
4461
4462 void handleMaskedCompressStore(IntrinsicInst &I) {
4463 IRBuilder<> IRB(&I);
4464 Value *Values = I.getArgOperand(i: 0);
4465 Value *Ptr = I.getArgOperand(i: 1);
4466 MaybeAlign Align = I.getParamAlign(ArgNo: 1);
4467 Value *Mask = I.getArgOperand(i: 2);
4468
4469 if (ClCheckAccessAddress) {
4470 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4471 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4472 }
4473
4474 Value *Shadow = getShadow(V: Values);
4475 Type *ElementShadowTy =
4476 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4477 auto [ShadowPtr, OriginPtrs] =
4478 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ true);
4479
4480 IRB.CreateMaskedCompressStore(Val: Shadow, Ptr: ShadowPtr, Align, Mask);
4481
4482 // TODO: Store origins.
4483 }
4484
4485 void handleMaskedGather(IntrinsicInst &I) {
4486 IRBuilder<> IRB(&I);
4487 Value *Ptrs = I.getArgOperand(i: 0);
4488 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4489 Value *Mask = I.getArgOperand(i: 1);
4490 Value *PassThru = I.getArgOperand(i: 2);
4491
4492 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4493 if (ClCheckAccessAddress) {
4494 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4495 Value *MaskedPtrShadow = IRB.CreateSelect(
4496 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4497 Name: "_msmaskedptrs");
4498 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4499 }
4500
4501 if (!PropagateShadow) {
4502 setShadow(V: &I, SV: getCleanShadow(V: &I));
4503 setOrigin(V: &I, Origin: getCleanOrigin());
4504 return;
4505 }
4506
4507 Type *ShadowTy = getShadowTy(V: &I);
4508 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4509 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4510 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ false);
4511
4512 Value *Shadow =
4513 IRB.CreateMaskedGather(Ty: ShadowTy, Ptrs: ShadowPtrs, Alignment, Mask,
4514 PassThru: getShadow(V: PassThru), Name: "_msmaskedgather");
4515
4516 setShadow(V: &I, SV: Shadow);
4517
4518 // TODO: Store origins.
4519 setOrigin(V: &I, Origin: getCleanOrigin());
4520 }
4521
4522 void handleMaskedScatter(IntrinsicInst &I) {
4523 IRBuilder<> IRB(&I);
4524 Value *Values = I.getArgOperand(i: 0);
4525 Value *Ptrs = I.getArgOperand(i: 1);
4526 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4527 Value *Mask = I.getArgOperand(i: 2);
4528
4529 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4530 if (ClCheckAccessAddress) {
4531 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4532 Value *MaskedPtrShadow = IRB.CreateSelect(
4533 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4534 Name: "_msmaskedptrs");
4535 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4536 }
4537
4538 Value *Shadow = getShadow(V: Values);
4539 Type *ElementShadowTy =
4540 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4541 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4542 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ true);
4543
4544 IRB.CreateMaskedScatter(Val: Shadow, Ptrs: ShadowPtrs, Alignment, Mask);
4545
4546 // TODO: Store origin.
4547 }
4548
4549 // Intrinsic::masked_store
4550 //
4551 // Note: handleAVXMaskedStore handles AVX/AVX2 variants, though AVX512 masked
4552 // stores are lowered to Intrinsic::masked_store.
4553 void handleMaskedStore(IntrinsicInst &I) {
4554 IRBuilder<> IRB(&I);
4555 Value *V = I.getArgOperand(i: 0);
4556 Value *Ptr = I.getArgOperand(i: 1);
4557 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4558 Value *Mask = I.getArgOperand(i: 2);
4559 Value *Shadow = getShadow(V);
4560
4561 if (ClCheckAccessAddress) {
4562 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4563 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4564 }
4565
4566 Value *ShadowPtr;
4567 Value *OriginPtr;
4568 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
4569 Addr: Ptr, IRB, ShadowTy: Shadow->getType(), Alignment, /*isStore*/ true);
4570
4571 IRB.CreateMaskedStore(Val: Shadow, Ptr: ShadowPtr, Alignment, Mask);
4572
4573 if (!MS.TrackOrigins)
4574 return;
4575
4576 auto &DL = F.getDataLayout();
4577 paintOrigin(IRB, Origin: getOrigin(V), OriginPtr,
4578 TS: DL.getTypeStoreSize(Ty: Shadow->getType()),
4579 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4580 }
4581
4582 // Intrinsic::masked_load
4583 //
4584 // Note: handleAVXMaskedLoad handles AVX/AVX2 variants, though AVX512 masked
4585 // loads are lowered to Intrinsic::masked_load.
4586 void handleMaskedLoad(IntrinsicInst &I) {
4587 IRBuilder<> IRB(&I);
4588 Value *Ptr = I.getArgOperand(i: 0);
4589 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4590 Value *Mask = I.getArgOperand(i: 1);
4591 Value *PassThru = I.getArgOperand(i: 2);
4592
4593 if (ClCheckAccessAddress) {
4594 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4595 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4596 }
4597
4598 if (!PropagateShadow) {
4599 setShadow(V: &I, SV: getCleanShadow(V: &I));
4600 setOrigin(V: &I, Origin: getCleanOrigin());
4601 return;
4602 }
4603
4604 Type *ShadowTy = getShadowTy(V: &I);
4605 Value *ShadowPtr, *OriginPtr;
4606 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4607 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
4608 setShadow(V: &I, SV: IRB.CreateMaskedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Alignment, Mask,
4609 PassThru: getShadow(V: PassThru), Name: "_msmaskedld"));
4610
4611 if (!MS.TrackOrigins)
4612 return;
4613
4614 // Choose between PassThru's and the loaded value's origins.
4615 Value *MaskedPassThruShadow = IRB.CreateAnd(
4616 LHS: getShadow(V: PassThru), RHS: IRB.CreateSExt(V: IRB.CreateNeg(V: Mask), DestTy: ShadowTy));
4617
4618 Value *NotNull = convertToBool(V: MaskedPassThruShadow, IRB, name: "_mscmp");
4619
4620 Value *PtrOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr);
4621 Value *Origin = IRB.CreateSelect(C: NotNull, True: getOrigin(V: PassThru), False: PtrOrigin);
4622
4623 setOrigin(V: &I, Origin);
4624 }
4625
4626 // e.g., void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)
4627 // dst mask src
4628 //
4629 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4630 // by handleMaskedStore.
4631 //
4632 // This function handles AVX and AVX2 masked stores; these use the MSBs of a
4633 // vector of integers, unlike the LLVM masked intrinsics, which require a
4634 // vector of booleans. X86InstCombineIntrinsic.cpp::simplifyX86MaskedLoad
4635 // mentions that the x86 backend does not know how to efficiently convert
4636 // from a vector of booleans back into the AVX mask format; therefore, they
4637 // (and we) do not reduce AVX/AVX2 masked intrinsics into LLVM masked
4638 // intrinsics.
4639 void handleAVXMaskedStore(IntrinsicInst &I) {
4640 assert(I.arg_size() == 3);
4641
4642 IRBuilder<> IRB(&I);
4643
4644 Value *Dst = I.getArgOperand(i: 0);
4645 assert(Dst->getType()->isPointerTy() && "Destination is not a pointer!");
4646
4647 Value *Mask = I.getArgOperand(i: 1);
4648 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4649
4650 Value *Src = I.getArgOperand(i: 2);
4651 assert(isa<VectorType>(Src->getType()) && "Source is not a vector!");
4652
4653 const Align Alignment = Align(1);
4654
4655 Value *SrcShadow = getShadow(V: Src);
4656
4657 if (ClCheckAccessAddress) {
4658 insertCheckShadowOf(Val: Dst, OrigIns: &I);
4659 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4660 }
4661
4662 Value *DstShadowPtr;
4663 Value *DstOriginPtr;
4664 std::tie(args&: DstShadowPtr, args&: DstOriginPtr) = getShadowOriginPtr(
4665 Addr: Dst, IRB, ShadowTy: SrcShadow->getType(), Alignment, /*isStore*/ true);
4666
4667 SmallVector<Value *, 2> ShadowArgs;
4668 ShadowArgs.append(NumInputs: 1, Elt: DstShadowPtr);
4669 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4670 // The intrinsic may require floating-point but shadows can be arbitrary
4671 // bit patterns, of which some would be interpreted as "invalid"
4672 // floating-point values (NaN etc.); we assume the intrinsic will happily
4673 // copy them.
4674 ShadowArgs.append(NumInputs: 1, Elt: IRB.CreateBitCast(V: SrcShadow, DestTy: Src->getType()));
4675
4676 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4677 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4678 setShadow(V: &I, SV: CI);
4679
4680 if (!MS.TrackOrigins)
4681 return;
4682
4683 // Approximation only
4684 auto &DL = F.getDataLayout();
4685 paintOrigin(IRB, Origin: getOrigin(V: Src), OriginPtr: DstOriginPtr,
4686 TS: DL.getTypeStoreSize(Ty: SrcShadow->getType()),
4687 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4688 }
4689
4690 // e.g., <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)
4691 // return src mask
4692 //
4693 // Masked-off values are replaced with 0, which conveniently also represents
4694 // initialized memory.
4695 //
4696 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4697 // by handleMaskedStore.
4698 //
4699 // We do not combine this with handleMaskedLoad; see comment in
4700 // handleAVXMaskedStore for the rationale.
4701 //
4702 // This is subtly different than handleIntrinsicByApplyingToShadow(I, 1)
4703 // because we need to apply getShadowOriginPtr, not getShadow, to the first
4704 // parameter.
4705 void handleAVXMaskedLoad(IntrinsicInst &I) {
4706 assert(I.arg_size() == 2);
4707
4708 IRBuilder<> IRB(&I);
4709
4710 Value *Src = I.getArgOperand(i: 0);
4711 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
4712
4713 Value *Mask = I.getArgOperand(i: 1);
4714 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4715
4716 const Align Alignment = Align(1);
4717
4718 if (ClCheckAccessAddress) {
4719 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4720 }
4721
4722 Type *SrcShadowTy = getShadowTy(V: Src);
4723 Value *SrcShadowPtr, *SrcOriginPtr;
4724 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
4725 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment, /*isStore*/ false);
4726
4727 SmallVector<Value *, 2> ShadowArgs;
4728 ShadowArgs.append(NumInputs: 1, Elt: SrcShadowPtr);
4729 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4730
4731 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4732 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4733 // The AVX masked load intrinsics do not have integer variants. We use the
4734 // floating-point variants, which will happily copy the shadows even if
4735 // they are interpreted as "invalid" floating-point values (NaN etc.).
4736 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4737
4738 if (!MS.TrackOrigins)
4739 return;
4740
4741 // The "pass-through" value is always zero (initialized). To the extent
4742 // that that results in initialized aligned 4-byte chunks, the origin value
4743 // is ignored. It is therefore correct to simply copy the origin from src.
4744 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
4745 setOrigin(V: &I, Origin: PtrSrcOrigin);
4746 }
4747
4748 // Test whether the mask indices are initialized, only checking the bits that
4749 // are actually used.
4750 //
4751 // e.g., if Idx is <32 x i16>, only (log2(32) == 5) bits of each index are
4752 // used/checked.
4753 void maskedCheckAVXIndexShadow(IRBuilder<> &IRB, Value *Idx, Instruction *I) {
4754 assert(isFixedIntVector(Idx));
4755 auto IdxVectorSize =
4756 cast<FixedVectorType>(Val: Idx->getType())->getNumElements();
4757 assert(isPowerOf2_64(IdxVectorSize));
4758
4759 // Compiler isn't smart enough, let's help it
4760 if (isa<Constant>(Val: Idx))
4761 return;
4762
4763 auto *IdxShadow = getShadow(V: Idx);
4764 Value *Truncated = IRB.CreateTrunc(
4765 V: IdxShadow,
4766 DestTy: FixedVectorType::get(ElementType: Type::getIntNTy(C&: *MS.C, N: Log2_64(Value: IdxVectorSize)),
4767 NumElts: IdxVectorSize));
4768 insertCheckShadow(Shadow: Truncated, Origin: getOrigin(V: Idx), OrigIns: I);
4769 }
4770
4771 // Instrument AVX permutation intrinsic.
4772 // We apply the same permutation (argument index 1) to the shadow.
4773 void handleAVXVpermilvar(IntrinsicInst &I) {
4774 IRBuilder<> IRB(&I);
4775 Value *Shadow = getShadow(I: &I, i: 0);
4776 maskedCheckAVXIndexShadow(IRB, Idx: I.getArgOperand(i: 1), I: &I);
4777
4778 // Shadows are integer-ish types but some intrinsics require a
4779 // different (e.g., floating-point) type.
4780 Shadow = IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i: 0)->getType());
4781 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4782 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Shadow, I.getArgOperand(i: 1)});
4783
4784 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4785 setOriginForNaryOp(I);
4786 }
4787
4788 // Instrument AVX permutation intrinsic.
4789 // We apply the same permutation (argument index 1) to the shadows.
4790 void handleAVXVpermi2var(IntrinsicInst &I) {
4791 assert(I.arg_size() == 3);
4792 assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
4793 assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
4794 assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
4795 [[maybe_unused]] auto ArgVectorSize =
4796 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4797 assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
4798 ->getNumElements() == ArgVectorSize);
4799 assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
4800 ->getNumElements() == ArgVectorSize);
4801 assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
4802 assert(I.getType() == I.getArgOperand(0)->getType());
4803 assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
4804 IRBuilder<> IRB(&I);
4805 Value *AShadow = getShadow(I: &I, i: 0);
4806 Value *Idx = I.getArgOperand(i: 1);
4807 Value *BShadow = getShadow(I: &I, i: 2);
4808
4809 maskedCheckAVXIndexShadow(IRB, Idx, I: &I);
4810
4811 // Shadows are integer-ish types but some intrinsics require a
4812 // different (e.g., floating-point) type.
4813 AShadow = IRB.CreateBitCast(V: AShadow, DestTy: I.getArgOperand(i: 0)->getType());
4814 BShadow = IRB.CreateBitCast(V: BShadow, DestTy: I.getArgOperand(i: 2)->getType());
4815 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4816 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {AShadow, Idx, BShadow});
4817 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4818 setOriginForNaryOp(I);
4819 }
4820
4821 [[maybe_unused]] static bool isFixedIntVectorTy(const Type *T) {
4822 return isa<FixedVectorType>(Val: T) && T->isIntOrIntVectorTy();
4823 }
4824
4825 [[maybe_unused]] static bool isFixedFPVectorTy(const Type *T) {
4826 return isa<FixedVectorType>(Val: T) && T->isFPOrFPVectorTy();
4827 }
4828
4829 [[maybe_unused]] static bool isFixedIntVector(const Value *V) {
4830 return isFixedIntVectorTy(T: V->getType());
4831 }
4832
4833 [[maybe_unused]] static bool isFixedFPVector(const Value *V) {
4834 return isFixedFPVectorTy(T: V->getType());
4835 }
4836
4837 // e.g., <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
4838 // (<16 x float> a, <16 x i32> writethru, i16 mask,
4839 // i32 rounding)
4840 //
4841 // Inconveniently, some similar intrinsics have a different operand order:
4842 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
4843 // (<16 x float> a, i32 rounding, <16 x i16> writethru,
4844 // i16 mask)
4845 //
4846 // If the return type has more elements than A, the excess elements are
4847 // zeroed (and the corresponding shadow is initialized).
4848 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
4849 // (<4 x float> a, i32 rounding, <8 x i16> writethru,
4850 // i8 mask)
4851 //
4852 // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
4853 // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
4854 // where all_or_nothing(x) is fully uninitialized if x has any
4855 // uninitialized bits
4856 void handleAVX512VectorConvertFPToInt(IntrinsicInst &I, bool LastMask) {
4857 IRBuilder<> IRB(&I);
4858
4859 assert(I.arg_size() == 4);
4860 Value *A = I.getOperand(i_nocapture: 0);
4861 Value *WriteThrough;
4862 Value *Mask;
4863 Value *RoundingMode;
4864 if (LastMask) {
4865 WriteThrough = I.getOperand(i_nocapture: 2);
4866 Mask = I.getOperand(i_nocapture: 3);
4867 RoundingMode = I.getOperand(i_nocapture: 1);
4868 } else {
4869 WriteThrough = I.getOperand(i_nocapture: 1);
4870 Mask = I.getOperand(i_nocapture: 2);
4871 RoundingMode = I.getOperand(i_nocapture: 3);
4872 }
4873
4874 assert(isFixedFPVector(A));
4875 assert(isFixedIntVector(WriteThrough));
4876
4877 unsigned ANumElements =
4878 cast<FixedVectorType>(Val: A->getType())->getNumElements();
4879 [[maybe_unused]] unsigned WriteThruNumElements =
4880 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
4881 assert(ANumElements == WriteThruNumElements ||
4882 ANumElements * 2 == WriteThruNumElements);
4883
4884 assert(Mask->getType()->isIntegerTy());
4885 unsigned MaskNumElements = Mask->getType()->getScalarSizeInBits();
4886 assert(ANumElements == MaskNumElements ||
4887 ANumElements * 2 == MaskNumElements);
4888
4889 assert(WriteThruNumElements == MaskNumElements);
4890
4891 // Some bits of the mask may be unused, though it's unusual to have partly
4892 // uninitialized bits.
4893 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4894
4895 assert(RoundingMode->getType()->isIntegerTy());
4896 // Only some bits of the rounding mode are used, though it's very
4897 // unusual to have uninitialized bits there (more commonly, it's a
4898 // constant).
4899 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
4900
4901 assert(I.getType() == WriteThrough->getType());
4902
4903 Value *AShadow = getShadow(V: A);
4904 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
4905
4906 if (ANumElements * 2 == MaskNumElements) {
4907 // Ensure that the irrelevant bits of the mask are zero, hence selecting
4908 // from the zeroed shadow instead of the writethrough's shadow.
4909 Mask =
4910 IRB.CreateTrunc(V: Mask, DestTy: IRB.getIntNTy(N: ANumElements), Name: "_ms_mask_trunc");
4911 Mask =
4912 IRB.CreateZExt(V: Mask, DestTy: IRB.getIntNTy(N: MaskNumElements), Name: "_ms_mask_zext");
4913 }
4914
4915 // Convert i16 mask to <16 x i1>
4916 Mask = IRB.CreateBitCast(
4917 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: MaskNumElements),
4918 Name: "_ms_mask_bitcast");
4919
4920 /// For floating-point to integer conversion, the output is:
4921 /// - fully uninitialized if *any* bit of the input is uninitialized
4922 /// - fully ininitialized if all bits of the input are ininitialized
4923 /// We apply the same principle on a per-element basis for vectors.
4924 ///
4925 /// We use the scalar width of the return type instead of A's.
4926 AShadow = IRB.CreateSExt(
4927 V: IRB.CreateICmpNE(LHS: AShadow, RHS: getCleanShadow(OrigTy: AShadow->getType())),
4928 DestTy: getShadowTy(V: &I), Name: "_ms_a_shadow");
4929
4930 Value *WriteThroughShadow = getShadow(V: WriteThrough);
4931 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow,
4932 Name: "_ms_writethru_select");
4933
4934 setShadow(V: &I, SV: Shadow);
4935 setOriginForNaryOp(I);
4936 }
4937
4938 static SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
4939 SmallVector<int, 8> Mask;
4940 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
4941 Mask.append(NumInputs: 2, Elt: X);
4942 }
4943 return Mask;
4944 }
4945
4946 // Instrument pclmul intrinsics.
4947 // These intrinsics operate either on odd or on even elements of the input
4948 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
4949 // Replace the unused elements with copies of the used ones, ex:
4950 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
4951 // or
4952 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
4953 // and then apply the usual shadow combining logic.
4954 void handlePclmulIntrinsic(IntrinsicInst &I) {
4955 IRBuilder<> IRB(&I);
4956 unsigned Width =
4957 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4958 assert(isa<ConstantInt>(I.getArgOperand(2)) &&
4959 "pclmul 3rd operand must be a constant");
4960 unsigned Imm = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4961 Value *Shuf0 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 0),
4962 Mask: getPclmulMask(Width, OddElements: Imm & 0x01));
4963 Value *Shuf1 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 1),
4964 Mask: getPclmulMask(Width, OddElements: Imm & 0x10));
4965 ShadowAndOriginCombiner SOC(this, IRB);
4966 SOC.Add(OpShadow: Shuf0, OpOrigin: getOrigin(I: &I, i: 0));
4967 SOC.Add(OpShadow: Shuf1, OpOrigin: getOrigin(I: &I, i: 1));
4968 SOC.Done(I: &I);
4969 }
4970
4971 // Instrument _mm_*_sd|ss intrinsics
4972 void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
4973 IRBuilder<> IRB(&I);
4974 unsigned Width =
4975 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4976 Value *First = getShadow(I: &I, i: 0);
4977 Value *Second = getShadow(I: &I, i: 1);
4978 // First element of second operand, remaining elements of first operand
4979 SmallVector<int, 16> Mask;
4980 Mask.push_back(Elt: Width);
4981 for (unsigned i = 1; i < Width; i++)
4982 Mask.push_back(Elt: i);
4983 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: Second, Mask);
4984
4985 setShadow(V: &I, SV: Shadow);
4986 setOriginForNaryOp(I);
4987 }
4988
4989 void handleVtestIntrinsic(IntrinsicInst &I) {
4990 IRBuilder<> IRB(&I);
4991 Value *Shadow0 = getShadow(I: &I, i: 0);
4992 Value *Shadow1 = getShadow(I: &I, i: 1);
4993 Value *Or = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4994 Value *NZ = IRB.CreateICmpNE(LHS: Or, RHS: Constant::getNullValue(Ty: Or->getType()));
4995 Value *Scalar = convertShadowToScalar(V: NZ, IRB);
4996 Value *Shadow = IRB.CreateZExt(V: Scalar, DestTy: getShadowTy(V: &I));
4997
4998 setShadow(V: &I, SV: Shadow);
4999 setOriginForNaryOp(I);
5000 }
5001
5002 void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
5003 IRBuilder<> IRB(&I);
5004 unsigned Width =
5005 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
5006 Value *First = getShadow(I: &I, i: 0);
5007 Value *Second = getShadow(I: &I, i: 1);
5008 Value *OrShadow = IRB.CreateOr(LHS: First, RHS: Second);
5009 // First element of both OR'd together, remaining elements of first operand
5010 SmallVector<int, 16> Mask;
5011 Mask.push_back(Elt: Width);
5012 for (unsigned i = 1; i < Width; i++)
5013 Mask.push_back(Elt: i);
5014 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: OrShadow, Mask);
5015
5016 setShadow(V: &I, SV: Shadow);
5017 setOriginForNaryOp(I);
5018 }
5019
5020 // _mm_round_ps / _mm_round_ps.
5021 // Similar to maybeHandleSimpleNomemIntrinsic except
5022 // the second argument is guaranteed to be a constant integer.
5023 void handleRoundPdPsIntrinsic(IntrinsicInst &I) {
5024 assert(I.getArgOperand(0)->getType() == I.getType());
5025 assert(I.arg_size() == 2);
5026 assert(isa<ConstantInt>(I.getArgOperand(1)));
5027
5028 IRBuilder<> IRB(&I);
5029 ShadowAndOriginCombiner SC(this, IRB);
5030 SC.Add(V: I.getArgOperand(i: 0));
5031 SC.Done(I: &I);
5032 }
5033
5034 // Instrument @llvm.abs intrinsic.
5035 //
5036 // e.g., i32 @llvm.abs.i32 (i32 <Src>, i1 <is_int_min_poison>)
5037 // <4 x i32> @llvm.abs.v4i32(<4 x i32> <Src>, i1 <is_int_min_poison>)
5038 void handleAbsIntrinsic(IntrinsicInst &I) {
5039 assert(I.arg_size() == 2);
5040 Value *Src = I.getArgOperand(i: 0);
5041 Value *IsIntMinPoison = I.getArgOperand(i: 1);
5042
5043 assert(I.getType()->isIntOrIntVectorTy());
5044
5045 assert(Src->getType() == I.getType());
5046
5047 assert(IsIntMinPoison->getType()->isIntegerTy());
5048 assert(IsIntMinPoison->getType()->getIntegerBitWidth() == 1);
5049
5050 IRBuilder<> IRB(&I);
5051 Value *SrcShadow = getShadow(V: Src);
5052
5053 APInt MinVal =
5054 APInt::getSignedMinValue(numBits: Src->getType()->getScalarSizeInBits());
5055 Value *MinValVec = ConstantInt::get(Ty: Src->getType(), V: MinVal);
5056 Value *SrcIsMin = IRB.CreateICmp(P: CmpInst::ICMP_EQ, LHS: Src, RHS: MinValVec);
5057
5058 Value *PoisonedShadow = getPoisonedShadow(V: Src);
5059 Value *PoisonedIfIntMinShadow =
5060 IRB.CreateSelect(C: SrcIsMin, True: PoisonedShadow, False: SrcShadow);
5061 Value *Shadow =
5062 IRB.CreateSelect(C: IsIntMinPoison, True: PoisonedIfIntMinShadow, False: SrcShadow);
5063
5064 setShadow(V: &I, SV: Shadow);
5065 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5066 }
5067
5068 void handleIsFpClass(IntrinsicInst &I) {
5069 IRBuilder<> IRB(&I);
5070 Value *Shadow = getShadow(I: &I, i: 0);
5071 setShadow(V: &I, SV: IRB.CreateICmpNE(LHS: Shadow, RHS: getCleanShadow(V: Shadow)));
5072 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5073 }
5074
5075 void handleArithmeticWithOverflow(IntrinsicInst &I) {
5076 IRBuilder<> IRB(&I);
5077 Value *Shadow0 = getShadow(I: &I, i: 0);
5078 Value *Shadow1 = getShadow(I: &I, i: 1);
5079 Value *ShadowElt0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
5080 Value *ShadowElt1 =
5081 IRB.CreateICmpNE(LHS: ShadowElt0, RHS: getCleanShadow(V: ShadowElt0));
5082
5083 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5084 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt0, Idxs: 0);
5085 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt1, Idxs: 1);
5086
5087 setShadow(V: &I, SV: Shadow);
5088 setOriginForNaryOp(I);
5089 }
5090
5091 Value *extractLowerShadow(IRBuilder<> &IRB, Value *V) {
5092 assert(isa<FixedVectorType>(V->getType()));
5093 assert(cast<FixedVectorType>(V->getType())->getNumElements() > 0);
5094 Value *Shadow = getShadow(V);
5095 return IRB.CreateExtractElement(Vec: Shadow,
5096 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5097 }
5098
5099 // Handle llvm.x86.avx512.mask.pmov{,s,us}.*.{128,256,512}
5100 //
5101 // e.g., call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512
5102 // (<8 x i64>, <16 x i8>, i8)
5103 // A WriteThru Mask
5104 //
5105 // call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512
5106 // (<16 x i32>, <16 x i8>, i16)
5107 //
5108 // Dst[i] = Mask[i] ? truncate_or_saturate(A[i]) : WriteThru[i]
5109 // Dst_shadow[i] = Mask[i] ? truncate(A_shadow[i]) : WriteThru_shadow[i]
5110 //
5111 // If Dst has more elements than A, the excess elements are zeroed (and the
5112 // corresponding shadow is initialized).
5113 //
5114 // Note: for PMOV (truncation), handleIntrinsicByApplyingToShadow is precise
5115 // and is much faster than this handler.
5116 void handleAVX512VectorDownConvert(IntrinsicInst &I) {
5117 IRBuilder<> IRB(&I);
5118
5119 assert(I.arg_size() == 3);
5120 Value *A = I.getOperand(i_nocapture: 0);
5121 Value *WriteThrough = I.getOperand(i_nocapture: 1);
5122 Value *Mask = I.getOperand(i_nocapture: 2);
5123
5124 assert(isFixedIntVector(A));
5125 assert(isFixedIntVector(WriteThrough));
5126
5127 unsigned ANumElements =
5128 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5129 unsigned OutputNumElements =
5130 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
5131 assert(ANumElements == OutputNumElements ||
5132 ANumElements * 2 == OutputNumElements);
5133 // N.B. some PMOV{,S,US} instructions have a 4x or even 8x ratio in the
5134 // number of elements e.g.,
5135 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256
5136 // (<4 x i64>, <16 x i8>, i8)
5137 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128
5138 // (<2 x i64>, <16 x i8>, i8)
5139 // However, we currently handle those elsewhere.
5140
5141 assert(Mask->getType()->isIntegerTy());
5142 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5143
5144 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5145 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5146 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5147 assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
5148
5149 assert(I.getType() == WriteThrough->getType());
5150
5151 // Widen the mask, if necessary, to have one bit per element of the output
5152 // vector.
5153 // We want the extra bits to have '1's, so that the CreateSelect will
5154 // select the values from AShadow instead of WriteThroughShadow ("maskless"
5155 // versions of the intrinsics are sometimes implemented using an all-1's
5156 // mask and an undefined value for WriteThroughShadow). We accomplish this
5157 // by using bitwise NOT before and after the ZExt.
5158 if (ANumElements != OutputNumElements) {
5159 Mask = IRB.CreateNot(V: Mask);
5160 Mask = IRB.CreateZExt(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements),
5161 Name: "_ms_widen_mask");
5162 Mask = IRB.CreateNot(V: Mask);
5163 }
5164 Mask = IRB.CreateBitCast(
5165 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5166
5167 Value *AShadow = getShadow(V: A);
5168
5169 // The return type might have more elements than the input.
5170 // Temporarily shrink the return type's number of elements.
5171 VectorType *ShadowType = maybeShrinkVectorShadowType(Src: A, I);
5172
5173 // PMOV truncates; PMOVS/PMOVUS uses signed/unsigned saturation.
5174 // This handler treats them all as truncation, which leads to some rare
5175 // false positives in the cases where the truncated bytes could
5176 // unambiguously saturate the value e.g., if A = ??????10 ????????
5177 // (big-endian), the unsigned saturated byte conversion is 11111111 i.e.,
5178 // fully defined, but the truncated byte is ????????.
5179 //
5180 // TODO: use GetMinMaxUnsigned() to handle saturation precisely.
5181 AShadow = IRB.CreateTrunc(V: AShadow, DestTy: ShadowType, Name: "_ms_trunc_shadow");
5182 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
5183
5184 Value *WriteThroughShadow = getShadow(V: WriteThrough);
5185
5186 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow);
5187 setShadow(V: &I, SV: Shadow);
5188 setOriginForNaryOp(I);
5189 }
5190
5191 // Handle llvm.x86.avx512.* instructions that take vector(s) of floating-point
5192 // values and perform an operation whose shadow propagation should be handled
5193 // as all-or-nothing [*], with masking provided by a vector and a mask
5194 // supplied as an integer.
5195 //
5196 // [*] if all bits of a vector element are initialized, the output is fully
5197 // initialized; otherwise, the output is fully uninitialized
5198 //
5199 // e.g., <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
5200 // (<16 x float>, <16 x float>, i16)
5201 // A WriteThru Mask
5202 //
5203 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
5204 // (<2 x double>, <2 x double>, i8)
5205 // A WriteThru Mask
5206 //
5207 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
5208 // (<8 x double>, i32, <8 x double>, i8, i32)
5209 // A Imm WriteThru Mask Rounding
5210 //
5211 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
5212 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
5213 // WriteThru A B Mask Rnd
5214 //
5215 // All operands other than A, B, ..., and WriteThru (e.g., Mask, Imm,
5216 // Rounding) must be fully initialized.
5217 //
5218 // Dst[i] = Mask[i] ? some_op(A[i], B[i], ...)
5219 // : WriteThru[i]
5220 // Dst_shadow[i] = Mask[i] ? all_or_nothing(A_shadow[i] | B_shadow[i] | ...)
5221 // : WriteThru_shadow[i]
5222 void handleAVX512VectorGenericMaskedFP(IntrinsicInst &I,
5223 SmallVector<unsigned, 4> DataIndices,
5224 unsigned WriteThruIndex,
5225 unsigned MaskIndex) {
5226 IRBuilder<> IRB(&I);
5227
5228 unsigned NumArgs = I.arg_size();
5229
5230 assert(WriteThruIndex < NumArgs);
5231 assert(MaskIndex < NumArgs);
5232 assert(WriteThruIndex != MaskIndex);
5233 Value *WriteThru = I.getOperand(i_nocapture: WriteThruIndex);
5234
5235 unsigned OutputNumElements =
5236 cast<FixedVectorType>(Val: WriteThru->getType())->getNumElements();
5237
5238 assert(DataIndices.size() > 0);
5239
5240 bool isData[16] = {false};
5241 assert(NumArgs <= 16);
5242 for (unsigned i : DataIndices) {
5243 assert(i < NumArgs);
5244 assert(i != WriteThruIndex);
5245 assert(i != MaskIndex);
5246
5247 isData[i] = true;
5248
5249 Value *A = I.getOperand(i_nocapture: i);
5250 assert(isFixedFPVector(A));
5251 [[maybe_unused]] unsigned ANumElements =
5252 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5253 assert(ANumElements == OutputNumElements);
5254 }
5255
5256 Value *Mask = I.getOperand(i_nocapture: MaskIndex);
5257
5258 assert(isFixedFPVector(WriteThru));
5259
5260 for (unsigned i = 0; i < NumArgs; ++i) {
5261 if (!isData[i] && i != WriteThruIndex) {
5262 // Imm, Mask, Rounding etc. are "control" data, hence we require that
5263 // they be fully initialized.
5264 assert(I.getOperand(i)->getType()->isIntegerTy());
5265 insertCheckShadowOf(Val: I.getOperand(i_nocapture: i), OrigIns: &I);
5266 }
5267 }
5268
5269 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5270 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5271 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5272 assert(Mask->getType()->getScalarSizeInBits() == OutputNumElements);
5273
5274 assert(I.getType() == WriteThru->getType());
5275
5276 Mask = IRB.CreateBitCast(
5277 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5278
5279 Value *DataShadow = nullptr;
5280 for (unsigned i : DataIndices) {
5281 Value *A = I.getOperand(i_nocapture: i);
5282 if (DataShadow)
5283 DataShadow = IRB.CreateOr(LHS: DataShadow, RHS: getShadow(V: A));
5284 else
5285 DataShadow = getShadow(V: A);
5286 }
5287
5288 // All-or-nothing shadow
5289 DataShadow =
5290 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: DataShadow, RHS: getCleanShadow(V: DataShadow)),
5291 DestTy: DataShadow->getType());
5292
5293 Value *WriteThruShadow = getShadow(V: WriteThru);
5294
5295 Value *Shadow = IRB.CreateSelect(C: Mask, True: DataShadow, False: WriteThruShadow);
5296 setShadow(V: &I, SV: Shadow);
5297
5298 setOriginForNaryOp(I);
5299 }
5300
5301 // AVX512 Floating-Point Classification
5302 //
5303 // e.g.,
5304 // - < 8 x i1> @llvm.x86.avx512.fpclass.pd.512
5305 // (<8 x double> %input, i32 %classifiers)
5306 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512
5307 // (<16 x float> %input, i32 %classifiers)
5308 void handleAVX512FPClass(IntrinsicInst &I) {
5309 IRBuilder<> IRB(&I);
5310
5311 assert(I.arg_size() == 2);
5312
5313 Value *Input = I.getOperand(i_nocapture: 0);
5314 assert(isFixedFPVector(Input));
5315 [[maybe_unused]] FixedVectorType *InputType = cast<FixedVectorType>(Val: Input->getType());
5316
5317 Value *Classifiers = I.getOperand(i_nocapture: 1);
5318 assert(isa<ConstantInt>(Classifiers));
5319 // No shadow check needed for constants
5320
5321 assert(isFixedIntVectorTy(I.getType()));
5322 FixedVectorType *OutputType = cast<FixedVectorType>(Val: I.getType());
5323 assert(OutputType->getScalarSizeInBits() == 1);
5324
5325 assert(OutputType->getNumElements() == InputType->getNumElements());
5326
5327 Value *OutputShadow;
5328 if (cast<ConstantInt>(Val: Classifiers)->isZero())
5329 // Each bit specifies whether a particular classifier is enabled.
5330 // If Classifiers == 0, the output is trivially known to be zero, thus
5331 // the output is fully initialized.
5332 OutputShadow = getCleanShadow(OrigTy: OutputType);
5333 else
5334 // Approximate each bit of the output shadow based on whether the
5335 // corresponding input element is fully initialized. It is only
5336 // approximate because some classifications do not rely on all bits of
5337 // the input element.
5338 OutputShadow = IRB.CreateICmpNE(LHS: getShadow(V: Input), RHS: getCleanShadow(V: Input));
5339
5340 setShadow(V: &I, SV: OutputShadow);
5341
5342 setOriginForNaryOp(I);
5343 }
5344
5345 // For sh.* compiler intrinsics:
5346 // llvm.x86.avx512fp16.mask.{add/sub/mul/div/max/min}.sh.round
5347 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
5348 // A B WriteThru Mask RoundingMode
5349 //
5350 // DstShadow[0] = Mask[0] ? (AShadow[0] | BShadow[0]) : WriteThruShadow[0]
5351 // DstShadow[1..7] = AShadow[1..7]
5352 void visitGenericScalarHalfwordInst(IntrinsicInst &I) {
5353 IRBuilder<> IRB(&I);
5354
5355 assert(I.arg_size() == 5);
5356 Value *A = I.getOperand(i_nocapture: 0);
5357 Value *B = I.getOperand(i_nocapture: 1);
5358 Value *WriteThrough = I.getOperand(i_nocapture: 2);
5359 Value *Mask = I.getOperand(i_nocapture: 3);
5360 Value *RoundingMode = I.getOperand(i_nocapture: 4);
5361
5362 // Technically, we could probably just check whether the LSB is
5363 // initialized, but intuitively it feels like a partly uninitialized mask
5364 // is unintended, and we should warn the user immediately.
5365 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5366 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
5367
5368 assert(isa<FixedVectorType>(A->getType()));
5369 unsigned NumElements =
5370 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5371 assert(NumElements == 8);
5372 assert(A->getType() == B->getType());
5373 assert(B->getType() == WriteThrough->getType());
5374 assert(Mask->getType()->getPrimitiveSizeInBits() == NumElements);
5375 assert(RoundingMode->getType()->isIntegerTy());
5376
5377 Value *ALowerShadow = extractLowerShadow(IRB, V: A);
5378 Value *BLowerShadow = extractLowerShadow(IRB, V: B);
5379
5380 Value *ABLowerShadow = IRB.CreateOr(LHS: ALowerShadow, RHS: BLowerShadow);
5381
5382 Value *WriteThroughLowerShadow = extractLowerShadow(IRB, V: WriteThrough);
5383
5384 Mask = IRB.CreateBitCast(
5385 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: NumElements));
5386 Value *MaskLower =
5387 IRB.CreateExtractElement(Vec: Mask, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5388
5389 Value *AShadow = getShadow(V: A);
5390 Value *DstLowerShadow =
5391 IRB.CreateSelect(C: MaskLower, True: ABLowerShadow, False: WriteThroughLowerShadow);
5392 Value *DstShadow = IRB.CreateInsertElement(
5393 Vec: AShadow, NewElt: DstLowerShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0),
5394 Name: "_msprop");
5395
5396 setShadow(V: &I, SV: DstShadow);
5397 setOriginForNaryOp(I);
5398 }
5399
5400 // Approximately handle AVX Galois Field Affine Transformation
5401 //
5402 // e.g.,
5403 // <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
5404 // <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8>, <32 x i8>, i8)
5405 // <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
5406 // Out A x b
5407 // where A and x are packed matrices, b is a vector,
5408 // Out = A * x + b in GF(2)
5409 //
5410 // Multiplication in GF(2) is equivalent to bitwise AND. However, the matrix
5411 // computation also includes a parity calculation.
5412 //
5413 // For the bitwise AND of bits V1 and V2, the exact shadow is:
5414 // Out_Shadow = (V1_Shadow & V2_Shadow)
5415 // | (V1 & V2_Shadow)
5416 // | (V1_Shadow & V2 )
5417 //
5418 // We approximate the shadow of gf2p8affineqb using:
5419 // Out_Shadow = gf2p8affineqb(x_Shadow, A_shadow, 0)
5420 // | gf2p8affineqb(x, A_shadow, 0)
5421 // | gf2p8affineqb(x_Shadow, A, 0)
5422 // | set1_epi8(b_Shadow)
5423 //
5424 // This approximation has false negatives: if an intermediate dot-product
5425 // contains an even number of 1's, the parity is 0.
5426 // It has no false positives.
5427 void handleAVXGF2P8Affine(IntrinsicInst &I) {
5428 IRBuilder<> IRB(&I);
5429
5430 assert(I.arg_size() == 3);
5431 Value *A = I.getOperand(i_nocapture: 0);
5432 Value *X = I.getOperand(i_nocapture: 1);
5433 Value *B = I.getOperand(i_nocapture: 2);
5434
5435 assert(isFixedIntVector(A));
5436 assert(cast<VectorType>(A->getType())
5437 ->getElementType()
5438 ->getScalarSizeInBits() == 8);
5439
5440 assert(A->getType() == X->getType());
5441
5442 assert(B->getType()->isIntegerTy());
5443 assert(B->getType()->getScalarSizeInBits() == 8);
5444
5445 assert(I.getType() == A->getType());
5446
5447 Value *AShadow = getShadow(V: A);
5448 Value *XShadow = getShadow(V: X);
5449 Value *BZeroShadow = getCleanShadow(V: B);
5450
5451 Value *AShadowXShadow = IRB.CreateIntrinsic(
5452 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {XShadow, AShadow, BZeroShadow});
5453 Value *AShadowX = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5454 Args: {X, AShadow, BZeroShadow});
5455 Value *XShadowA = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5456 Args: {XShadow, A, BZeroShadow});
5457
5458 unsigned NumElements = cast<FixedVectorType>(Val: I.getType())->getNumElements();
5459 Value *BShadow = getShadow(V: B);
5460 Value *BBroadcastShadow = getCleanShadow(V: AShadow);
5461 // There is no LLVM IR intrinsic for _mm512_set1_epi8.
5462 // This loop generates a lot of LLVM IR, which we expect that CodeGen will
5463 // lower appropriately (e.g., VPBROADCASTB).
5464 // Besides, b is often a constant, in which case it is fully initialized.
5465 for (unsigned i = 0; i < NumElements; i++)
5466 BBroadcastShadow = IRB.CreateInsertElement(Vec: BBroadcastShadow, NewElt: BShadow, Idx: i);
5467
5468 setShadow(V: &I, SV: IRB.CreateOr(
5469 Ops: {AShadowXShadow, AShadowX, XShadowA, BBroadcastShadow}));
5470 setOriginForNaryOp(I);
5471 }
5472
5473 // Handle Arm NEON vector load intrinsics (vld*).
5474 //
5475 // The WithLane instructions (ld[234]lane) are similar to:
5476 // call {<4 x i32>, <4 x i32>, <4 x i32>}
5477 // @llvm.aarch64.neon.ld3lane.v4i32.p0
5478 // (<4 x i32> %L1, <4 x i32> %L2, <4 x i32> %L3, i64 %lane, ptr
5479 // %A)
5480 //
5481 // The non-WithLane instructions (ld[234], ld1x[234], ld[234]r) are similar
5482 // to:
5483 // call {<8 x i8>, <8 x i8>} @llvm.aarch64.neon.ld2.v8i8.p0(ptr %A)
5484 void handleNEONVectorLoad(IntrinsicInst &I, bool WithLane) {
5485 unsigned int numArgs = I.arg_size();
5486
5487 // Return type is a struct of vectors of integers or floating-point
5488 assert(I.getType()->isStructTy());
5489 [[maybe_unused]] StructType *RetTy = cast<StructType>(Val: I.getType());
5490 assert(RetTy->getNumElements() > 0);
5491 assert(RetTy->getElementType(0)->isIntOrIntVectorTy() ||
5492 RetTy->getElementType(0)->isFPOrFPVectorTy());
5493 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5494 assert(RetTy->getElementType(i) == RetTy->getElementType(0));
5495
5496 if (WithLane) {
5497 // 2, 3 or 4 vectors, plus lane number, plus input pointer
5498 assert(4 <= numArgs && numArgs <= 6);
5499
5500 // Return type is a struct of the input vectors
5501 assert(RetTy->getNumElements() + 2 == numArgs);
5502 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5503 assert(I.getArgOperand(i)->getType() == RetTy->getElementType(0));
5504 } else {
5505 assert(numArgs == 1);
5506 }
5507
5508 IRBuilder<> IRB(&I);
5509
5510 SmallVector<Value *, 6> ShadowArgs;
5511 if (WithLane) {
5512 for (unsigned int i = 0; i < numArgs - 2; i++)
5513 ShadowArgs.push_back(Elt: getShadow(V: I.getArgOperand(i)));
5514
5515 // Lane number, passed verbatim
5516 Value *LaneNumber = I.getArgOperand(i: numArgs - 2);
5517 ShadowArgs.push_back(Elt: LaneNumber);
5518
5519 // TODO: blend shadow of lane number into output shadow?
5520 insertCheckShadowOf(Val: LaneNumber, OrigIns: &I);
5521 }
5522
5523 Value *Src = I.getArgOperand(i: numArgs - 1);
5524 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
5525
5526 Type *SrcShadowTy = getShadowTy(V: Src);
5527 auto [SrcShadowPtr, SrcOriginPtr] =
5528 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment: Align(1), /*isStore*/ false);
5529 ShadowArgs.push_back(Elt: SrcShadowPtr);
5530
5531 // The NEON vector load instructions handled by this function all have
5532 // integer variants. It is easier to use those rather than trying to cast
5533 // a struct of vectors of floats into a struct of vectors of integers.
5534 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5535 RetTy: getShadowTy(V: &I), ID: I.getIntrinsicID(), Args: ShadowArgs);
5536 setShadow(V: &I, SV: CI);
5537
5538 if (!MS.TrackOrigins)
5539 return;
5540
5541 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
5542 setOrigin(V: &I, Origin: PtrSrcOrigin);
5543 }
5544
5545 /// Handle Arm NEON vector store intrinsics (vst{2,3,4}, vst1x_{2,3,4},
5546 /// and vst{2,3,4}lane).
5547 ///
5548 /// Arm NEON vector store intrinsics have the output address (pointer) as the
5549 /// last argument, with the initial arguments being the inputs (and lane
5550 /// number for vst{2,3,4}lane). They return void.
5551 ///
5552 /// - st4 interleaves the output e.g., st4 (inA, inB, inC, inD, outP) writes
5553 /// abcdabcdabcdabcd... into *outP
5554 /// - st1_x4 is non-interleaved e.g., st1_x4 (inA, inB, inC, inD, outP)
5555 /// writes aaaa...bbbb...cccc...dddd... into *outP
5556 /// - st4lane has arguments of (inA, inB, inC, inD, lane, outP)
5557 /// These instructions can all be instrumented with essentially the same
5558 /// MSan logic, simply by applying the corresponding intrinsic to the shadow.
5559 void handleNEONVectorStoreIntrinsic(IntrinsicInst &I, bool useLane) {
5560 IRBuilder<> IRB(&I);
5561
5562 // Don't use getNumOperands() because it includes the callee
5563 int numArgOperands = I.arg_size();
5564
5565 // The last arg operand is the output (pointer)
5566 assert(numArgOperands >= 1);
5567 Value *Addr = I.getArgOperand(i: numArgOperands - 1);
5568 assert(Addr->getType()->isPointerTy());
5569 int skipTrailingOperands = 1;
5570
5571 if (ClCheckAccessAddress)
5572 insertCheckShadowOf(Val: Addr, OrigIns: &I);
5573
5574 // Second-last operand is the lane number (for vst{2,3,4}lane)
5575 if (useLane) {
5576 skipTrailingOperands++;
5577 assert(numArgOperands >= static_cast<int>(skipTrailingOperands));
5578 assert(isa<IntegerType>(
5579 I.getArgOperand(numArgOperands - skipTrailingOperands)->getType()));
5580 }
5581
5582 SmallVector<Value *, 8> ShadowArgs;
5583 // All the initial operands are the inputs
5584 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++) {
5585 assert(isa<FixedVectorType>(I.getArgOperand(i)->getType()));
5586 Value *Shadow = getShadow(I: &I, i);
5587 ShadowArgs.append(NumInputs: 1, Elt: Shadow);
5588 }
5589
5590 // MSan's GetShadowTy assumes the LHS is the type we want the shadow for
5591 // e.g., for:
5592 // [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
5593 // we know the type of the output (and its shadow) is <16 x i8>.
5594 //
5595 // Arm NEON VST is unusual because the last argument is the output address:
5596 // define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) {
5597 // call void @llvm.aarch64.neon.st2.v16i8.p0
5598 // (<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
5599 // and we have no type information about P's operand. We must manually
5600 // compute the type (<16 x i8> x 2).
5601 FixedVectorType *OutputVectorTy = FixedVectorType::get(
5602 ElementType: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getElementType(),
5603 NumElts: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements() *
5604 (numArgOperands - skipTrailingOperands));
5605 Type *OutputShadowTy = getShadowTy(OrigTy: OutputVectorTy);
5606
5607 if (useLane)
5608 ShadowArgs.append(NumInputs: 1,
5609 Elt: I.getArgOperand(i: numArgOperands - skipTrailingOperands));
5610
5611 Value *OutputShadowPtr, *OutputOriginPtr;
5612 // AArch64 NEON does not need alignment (unless OS requires it)
5613 std::tie(args&: OutputShadowPtr, args&: OutputOriginPtr) = getShadowOriginPtr(
5614 Addr, IRB, ShadowTy: OutputShadowTy, Alignment: Align(1), /*isStore*/ true);
5615 ShadowArgs.append(NumInputs: 1, Elt: OutputShadowPtr);
5616
5617 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5618 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
5619 setShadow(V: &I, SV: CI);
5620
5621 if (MS.TrackOrigins) {
5622 // TODO: if we modelled the vst* instruction more precisely, we could
5623 // more accurately track the origins (e.g., if both inputs are
5624 // uninitialized for vst2, we currently blame the second input, even
5625 // though part of the output depends only on the first input).
5626 //
5627 // This is particularly imprecise for vst{2,3,4}lane, since only one
5628 // lane of each input is actually copied to the output.
5629 OriginCombiner OC(this, IRB);
5630 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++)
5631 OC.Add(V: I.getArgOperand(i));
5632
5633 const DataLayout &DL = F.getDataLayout();
5634 OC.DoneAndStoreOrigin(TS: DL.getTypeStoreSize(Ty: OutputVectorTy),
5635 OriginPtr: OutputOriginPtr);
5636 }
5637 }
5638
5639 // Integer matrix multiplication:
5640 // - <4 x i32> @llvm.aarch64.neon.{s,u,us}mmla.v4i32.v16i8
5641 // (<4 x i32> %R, <16 x i8> %A, <16 x i8> %B)
5642 // - <4 x i32> is a 2x2 matrix
5643 // - <16 x i8> %A and %B are 2x8 and 8x2 matrices respectively
5644 //
5645 // Floating-point matrix multiplication:
5646 // - <4 x float> @llvm.aarch64.neon.bfmmla
5647 // (<4 x float> %R, <8 x bfloat> %A, <8 x bfloat> %B)
5648 // - <4 x float> is a 2x2 matrix
5649 // - <8 x bfloat> %A and %B are 2x4 and 4x2 matrices respectively
5650 //
5651 // The general shadow propagation approach is:
5652 // 1) get the shadows of the input matrices %A and %B
5653 // 2) map each shadow value to 0x1 if the corresponding value is fully
5654 // initialized, and 0x0 otherwise
5655 // 3) perform a matrix multiplication on the shadows of %A and %B [*].
5656 // The output will be a 2x2 matrix. For each element, a value of 0x8
5657 // (for {s,u,us}mmla) or 0x4 (for bfmmla) means all the corresponding
5658 // inputs were clean; if so, set the shadow to zero, otherwise set to -1.
5659 // 4) blend in the shadow of %R
5660 //
5661 // [*] Since shadows are integral, the obvious approach is to always apply
5662 // ummla to the shadows. Unfortunately, Armv8.2+bf16 supports bfmmla,
5663 // but not ummla. Thus, for bfmmla, our instrumentation reuses bfmmla.
5664 //
5665 // TODO: consider allowing multiplication of zero with an uninitialized value
5666 // to result in an initialized value.
5667 void handleNEONMatrixMultiply(IntrinsicInst &I) {
5668 IRBuilder<> IRB(&I);
5669
5670 assert(I.arg_size() == 3);
5671 Value *R = I.getArgOperand(i: 0);
5672 Value *A = I.getArgOperand(i: 1);
5673 Value *B = I.getArgOperand(i: 2);
5674
5675 assert(I.getType() == R->getType());
5676
5677 assert(isa<FixedVectorType>(R->getType()));
5678 assert(isa<FixedVectorType>(A->getType()));
5679 assert(isa<FixedVectorType>(B->getType()));
5680
5681 FixedVectorType *RTy = cast<FixedVectorType>(Val: R->getType());
5682 FixedVectorType *ATy = cast<FixedVectorType>(Val: A->getType());
5683 FixedVectorType *BTy = cast<FixedVectorType>(Val: B->getType());
5684 assert(ATy->getElementType() == BTy->getElementType());
5685
5686 if (RTy->getElementType()->isIntegerTy()) {
5687 // <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8
5688 // (<4 x i32> %R, <16 x i8> %X, <16 x i8> %Y)
5689 assert(RTy == FixedVectorType::get(IntegerType::get(*MS.C, 32), 4));
5690 assert(ATy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5691 assert(BTy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5692 } else {
5693 // <4 x float> @llvm.aarch64.neon.bfmmla
5694 // (<4 x float> %R, <8 x bfloat> %X, <8 x bfloat> %Y)
5695 assert(RTy == FixedVectorType::get(Type::getFloatTy(*MS.C), 4));
5696 assert(ATy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5697 assert(BTy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5698 }
5699
5700 Value *ShadowR = getShadow(I: &I, i: 0);
5701 Value *ShadowA = getShadow(I: &I, i: 1);
5702 Value *ShadowB = getShadow(I: &I, i: 2);
5703
5704 Value *ShadowAB;
5705 Value *FullyInit;
5706
5707 if (RTy->getElementType()->isIntegerTy()) {
5708 // If the value is fully initialized, the shadow will be 000...001.
5709 // Otherwise, the shadow will be all zero.
5710 // (This is the opposite of how we typically handle shadows.)
5711 ShadowA = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5712 DestTy: getShadowTy(OrigTy: ATy));
5713 ShadowB = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5714 DestTy: getShadowTy(OrigTy: BTy));
5715 // TODO: the CreateSelect approach used below for floating-point is more
5716 // generic than CreateZExt. Investigate whether it is worthwhile
5717 // unifying the two approaches.
5718
5719 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_ummla,
5720 Args: {getCleanShadow(OrigTy: RTy), ShadowA, ShadowB});
5721
5722 // ummla multiplies a 2x8 matrix with an 8x2 matrix. If all entries of the
5723 // input matrices are equal to 0x1, all entries of the output matrix will
5724 // be 0x8.
5725 FullyInit = ConstantVector::getSplat(
5726 EC: RTy->getElementCount(), Elt: ConstantInt::get(Ty: RTy->getElementType(), V: 0x8));
5727
5728 ShadowAB = IRB.CreateICmpNE(LHS: ShadowAB, RHS: FullyInit);
5729 } else {
5730 Constant *ABZeros = ConstantVector::getSplat(
5731 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 0));
5732 Constant *ABOnes = ConstantVector::getSplat(
5733 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 1));
5734
5735 // As per the integer case, if the shadow is clean, we store 0x1,
5736 // otherwise we store 0x0 (the opposite of usual shadow arithmetic).
5737 ShadowA = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5738 True: ABOnes, False: ABZeros);
5739 ShadowB = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5740 True: ABOnes, False: ABZeros);
5741
5742 Constant *RZeros = ConstantVector::getSplat(
5743 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 0));
5744
5745 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_bfmmla,
5746 Args: {RZeros, ShadowA, ShadowB});
5747
5748 // bfmmla multiplies a 2x4 matrix with an 4x2 matrix. If all entries of
5749 // the input matrices are equal to 0x1, all entries of the output matrix
5750 // will be 4.0. (To avoid floating-point error, we check if each entry
5751 // < 3.5.)
5752 FullyInit = ConstantVector::getSplat(
5753 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 3.5));
5754
5755 // FCmpULT: "yields true if either operand is a QNAN or op1 is less than"
5756 // op2"
5757 ShadowAB = IRB.CreateFCmpULT(LHS: ShadowAB, RHS: FullyInit);
5758 }
5759
5760 ShadowR = IRB.CreateICmpNE(LHS: ShadowR, RHS: getCleanShadow(OrigTy: RTy));
5761 ShadowR = IRB.CreateOr(LHS: ShadowAB, RHS: ShadowR);
5762
5763 setShadow(V: &I, SV: IRB.CreateSExt(V: ShadowR, DestTy: getShadowTy(OrigTy: RTy)));
5764
5765 setOriginForNaryOp(I);
5766 }
5767
5768 /// Handle intrinsics by applying the intrinsic to the shadows.
5769 ///
5770 /// For example, this can be applied to the Arm NEON vector table intrinsics
5771 /// (tbl{1,2,3,4}).
5772 ///
5773 /// Typically, shadowIntrinsicID will be specified by the caller to be
5774 /// I.getIntrinsicID(), but the caller can choose to replace it with another
5775 /// intrinsic of the same type.
5776 ///
5777 /// The trailing arguments are passed verbatim to the intrinsic, though any
5778 /// uninitialized trailing arguments can also taint the shadow e.g., for an
5779 /// intrinsic with one trailing verbatim argument:
5780 /// out = intrinsic(var1, var2, opType)
5781 /// we compute:
5782 /// shadow[out] =
5783 /// intrinsic(shadow[var1], shadow[var2], opType) | shadow[opType]
5784 ///
5785 /// If an intrinsic is called with floating-point arguments, we will
5786 /// typically cast the shadows to floating-point, apply the intrinsic [*],
5787 /// then cast the result back to integer/shadow.
5788 ///
5789 /// In cases where we know the intrinsic is compatible with integer
5790 /// arguments, 'forceIntegerIntrinsic' will apply the integer variant, even
5791 /// if the arguments are floating-point, thus avoiding unnecessary casts
5792 /// e.g., if I is:
5793 /// <16 x float> @llvm.x86.avx512.mask.compress
5794 /// (<16 x float>, <16 x float>, <16 x i1> %mask)
5795 /// we would prefer to compute the shadows using:
5796 /// <16 x i32> @llvm.x86.avx512.mask.compress
5797 /// (<16 x i32>, <16 x i32>, <16 x i1> %mask)
5798 ///
5799 /// [*] CAUTION: this assumes that the intrinsic will handle arbitrary
5800 /// bit-patterns (for example, if the intrinsic accepts floats
5801 /// for var1, we require that it doesn't care if inputs are
5802 /// NaNs).
5803 ///
5804 /// The origin is approximated using setOriginForNaryOp.
5805 void handleIntrinsicByApplyingToShadow(IntrinsicInst &I,
5806 Intrinsic::ID shadowIntrinsicID,
5807 unsigned int trailingVerbatimArgs,
5808 bool forceIntegerIntrinsic) {
5809 IRBuilder<> IRB(&I);
5810
5811 assert(trailingVerbatimArgs < I.arg_size());
5812
5813 SmallVector<Value *, 8> ShadowArgs;
5814 // Don't use getNumOperands() because it includes the callee
5815 for (unsigned int i = 0; i < I.arg_size() - trailingVerbatimArgs; i++) {
5816 Value *Shadow = getShadow(I: &I, i);
5817
5818 if (forceIntegerIntrinsic)
5819 ShadowArgs.push_back(Elt: Shadow);
5820 else
5821 ShadowArgs.push_back(
5822 Elt: IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i)->getType()));
5823 }
5824
5825 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5826 i++) {
5827 Value *Arg = I.getArgOperand(i);
5828 if (forceIntegerIntrinsic)
5829 assert(Arg->getType()->isIntOrIntVectorTy());
5830 ShadowArgs.push_back(Elt: Arg);
5831 }
5832
5833 Value *CombinedShadow;
5834 if (forceIntegerIntrinsic) {
5835 CombinedShadow =
5836 IRB.CreateIntrinsic(RetTy: getShadowTy(V: &I), ID: shadowIntrinsicID, Args: ShadowArgs);
5837 } else {
5838 Value *CI =
5839 IRB.CreateIntrinsic(RetTy: I.getType(), ID: shadowIntrinsicID, Args: ShadowArgs);
5840 CombinedShadow = IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I));
5841 }
5842
5843 // Combine the computed shadow with the shadow of trailing args
5844 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5845 i++) {
5846 Value *Shadow =
5847 CreateShadowCast(IRB, V: getShadow(I: &I, i), dstTy: CombinedShadow->getType());
5848 CombinedShadow = IRB.CreateOr(LHS: Shadow, RHS: CombinedShadow, Name: "_msprop");
5849 }
5850
5851 setShadow(V: &I, SV: CombinedShadow);
5852
5853 setOriginForNaryOp(I);
5854 }
5855
5856 // Approximation only
5857 //
5858 // e.g., <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
5859 void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
5860 assert(I.arg_size() == 2);
5861
5862 handleShadowOr(I);
5863 }
5864
5865 bool maybeHandleCrossPlatformIntrinsic(IntrinsicInst &I) {
5866 switch (I.getIntrinsicID()) {
5867 case Intrinsic::uadd_with_overflow:
5868 case Intrinsic::sadd_with_overflow:
5869 case Intrinsic::usub_with_overflow:
5870 case Intrinsic::ssub_with_overflow:
5871 case Intrinsic::umul_with_overflow:
5872 case Intrinsic::smul_with_overflow:
5873 handleArithmeticWithOverflow(I);
5874 break;
5875 case Intrinsic::abs:
5876 handleAbsIntrinsic(I);
5877 break;
5878 case Intrinsic::bitreverse:
5879 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
5880 /*trailingVerbatimArgs=*/0,
5881 /*forceIntegerIntrinsic=*/false);
5882 break;
5883 case Intrinsic::is_fpclass:
5884 handleIsFpClass(I);
5885 break;
5886 case Intrinsic::lifetime_start:
5887 handleLifetimeStart(I);
5888 break;
5889 case Intrinsic::launder_invariant_group:
5890 case Intrinsic::strip_invariant_group:
5891 handleInvariantGroup(I);
5892 break;
5893 case Intrinsic::bswap:
5894 handleBswap(I);
5895 break;
5896 case Intrinsic::ctlz:
5897 case Intrinsic::cttz:
5898 handleCountLeadingTrailingZeros(I);
5899 break;
5900 case Intrinsic::masked_compressstore:
5901 handleMaskedCompressStore(I);
5902 break;
5903 case Intrinsic::masked_expandload:
5904 handleMaskedExpandLoad(I);
5905 break;
5906 case Intrinsic::masked_gather:
5907 handleMaskedGather(I);
5908 break;
5909 case Intrinsic::masked_scatter:
5910 handleMaskedScatter(I);
5911 break;
5912 case Intrinsic::masked_store:
5913 handleMaskedStore(I);
5914 break;
5915 case Intrinsic::masked_load:
5916 handleMaskedLoad(I);
5917 break;
5918 case Intrinsic::vector_reduce_and:
5919 handleVectorReduceAndIntrinsic(I);
5920 break;
5921 case Intrinsic::vector_reduce_or:
5922 handleVectorReduceOrIntrinsic(I);
5923 break;
5924
5925 case Intrinsic::vector_reduce_add:
5926 case Intrinsic::vector_reduce_xor:
5927 case Intrinsic::vector_reduce_mul:
5928 // Signed/Unsigned Min/Max
5929 // TODO: handling similarly to AND/OR may be more precise.
5930 case Intrinsic::vector_reduce_smax:
5931 case Intrinsic::vector_reduce_smin:
5932 case Intrinsic::vector_reduce_umax:
5933 case Intrinsic::vector_reduce_umin:
5934 // TODO: this has no false positives, but arguably we should check that all
5935 // the bits are initialized.
5936 case Intrinsic::vector_reduce_fmax:
5937 case Intrinsic::vector_reduce_fmin:
5938 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/false);
5939 break;
5940
5941 case Intrinsic::vector_reduce_fadd:
5942 case Intrinsic::vector_reduce_fmul:
5943 handleVectorReduceWithStarterIntrinsic(I);
5944 break;
5945
5946 case Intrinsic::scmp:
5947 case Intrinsic::ucmp: {
5948 handleShadowOr(I);
5949 break;
5950 }
5951
5952 case Intrinsic::fshl:
5953 case Intrinsic::fshr:
5954 handleFunnelShift(I);
5955 break;
5956
5957 case Intrinsic::pdep:
5958 case Intrinsic::pext:
5959 handleGenericBitManipulation(I);
5960 break;
5961
5962 case Intrinsic::is_constant:
5963 // The result of llvm.is.constant() is always defined.
5964 setShadow(V: &I, SV: getCleanShadow(V: &I));
5965 setOrigin(V: &I, Origin: getCleanOrigin());
5966 break;
5967
5968 // The non-saturating versions are handled by visitFPTo[US]IInst().
5969 //
5970 // N.B. some platform-specific intrinsics, such as AArch64 fcvtz[us], are
5971 // lowered to these cross-platform intrinsics.
5972 case Intrinsic::fptosi_sat:
5973 case Intrinsic::fptoui_sat:
5974 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
5975 break;
5976
5977 default:
5978 return false;
5979 }
5980
5981 return true;
5982 }
5983
5984 bool maybeHandleX86SIMDIntrinsic(IntrinsicInst &I) {
5985 switch (I.getIntrinsicID()) {
5986 case Intrinsic::x86_sse_stmxcsr:
5987 handleStmxcsr(I);
5988 break;
5989 case Intrinsic::x86_sse_ldmxcsr:
5990 handleLdmxcsr(I);
5991 break;
5992
5993 // Convert Scalar Double Precision Floating-Point Value
5994 // to Unsigned Doubleword Integer
5995 // etc.
5996 case Intrinsic::x86_avx512_vcvtsd2usi64:
5997 case Intrinsic::x86_avx512_vcvtsd2usi32:
5998 case Intrinsic::x86_avx512_vcvtss2usi64:
5999 case Intrinsic::x86_avx512_vcvtss2usi32:
6000 case Intrinsic::x86_avx512_cvttss2usi64:
6001 case Intrinsic::x86_avx512_cvttss2usi:
6002 case Intrinsic::x86_avx512_cvttsd2usi64:
6003 case Intrinsic::x86_avx512_cvttsd2usi:
6004 case Intrinsic::x86_avx512_cvtusi2ss:
6005 case Intrinsic::x86_avx512_cvtusi642sd:
6006 case Intrinsic::x86_avx512_cvtusi642ss:
6007 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1, HasRoundingMode: true);
6008 break;
6009 case Intrinsic::x86_sse2_cvtsd2si64:
6010 case Intrinsic::x86_sse2_cvtsd2si:
6011 case Intrinsic::x86_sse2_cvtsd2ss:
6012 case Intrinsic::x86_sse2_cvttsd2si64:
6013 case Intrinsic::x86_sse2_cvttsd2si:
6014 case Intrinsic::x86_sse_cvtss2si64:
6015 case Intrinsic::x86_sse_cvtss2si:
6016 case Intrinsic::x86_sse_cvttss2si64:
6017 case Intrinsic::x86_sse_cvttss2si:
6018 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1);
6019 break;
6020 case Intrinsic::x86_sse_cvtps2pi:
6021 case Intrinsic::x86_sse_cvttps2pi:
6022 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 2);
6023 break;
6024
6025 // TODO:
6026 // <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double>)
6027 // <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64>)
6028 // <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, <1 x i64>)
6029
6030 case Intrinsic::x86_vcvtps2ph_128:
6031 case Intrinsic::x86_vcvtps2ph_256: {
6032 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/true);
6033 break;
6034 }
6035
6036 // Convert Packed Single Precision Floating-Point Values
6037 // to Packed Signed Doubleword Integer Values
6038 //
6039 // <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
6040 // (<16 x float>, <16 x i32>, i16, i32)
6041 case Intrinsic::x86_avx512_mask_cvtps2dq_512:
6042 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/false);
6043 break;
6044
6045 // Convert Packed Double Precision Floating-Point Values
6046 // to Packed Single Precision Floating-Point Values
6047 case Intrinsic::x86_sse2_cvtpd2ps:
6048 case Intrinsic::x86_sse2_cvtps2dq:
6049 case Intrinsic::x86_sse2_cvtpd2dq:
6050 case Intrinsic::x86_sse2_cvttps2dq:
6051 case Intrinsic::x86_sse2_cvttpd2dq:
6052 case Intrinsic::x86_avx_cvt_pd2_ps_256:
6053 case Intrinsic::x86_avx_cvt_ps2dq_256:
6054 case Intrinsic::x86_avx_cvt_pd2dq_256:
6055 case Intrinsic::x86_avx_cvtt_ps2dq_256:
6056 case Intrinsic::x86_avx_cvtt_pd2dq_256: {
6057 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/false);
6058 break;
6059 }
6060
6061 // Convert Single-Precision FP Value to 16-bit FP Value
6062 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
6063 // (<16 x float>, i32, <16 x i16>, i16)
6064 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
6065 // (<4 x float>, i32, <8 x i16>, i8)
6066 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256
6067 // (<8 x float>, i32, <8 x i16>, i8)
6068 case Intrinsic::x86_avx512_mask_vcvtps2ph_512:
6069 case Intrinsic::x86_avx512_mask_vcvtps2ph_256:
6070 case Intrinsic::x86_avx512_mask_vcvtps2ph_128:
6071 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/true);
6072 break;
6073
6074 // Shift Packed Data (Left Logical, Right Arithmetic, Right Logical)
6075 case Intrinsic::x86_avx512_psll_w_512:
6076 case Intrinsic::x86_avx512_psll_d_512:
6077 case Intrinsic::x86_avx512_psll_q_512:
6078 case Intrinsic::x86_avx512_pslli_w_512:
6079 case Intrinsic::x86_avx512_pslli_d_512:
6080 case Intrinsic::x86_avx512_pslli_q_512:
6081 case Intrinsic::x86_avx512_psrl_w_512:
6082 case Intrinsic::x86_avx512_psrl_d_512:
6083 case Intrinsic::x86_avx512_psrl_q_512:
6084 case Intrinsic::x86_avx512_psra_w_512:
6085 case Intrinsic::x86_avx512_psra_d_512:
6086 case Intrinsic::x86_avx512_psra_q_512:
6087 case Intrinsic::x86_avx512_psrli_w_512:
6088 case Intrinsic::x86_avx512_psrli_d_512:
6089 case Intrinsic::x86_avx512_psrli_q_512:
6090 case Intrinsic::x86_avx512_psrai_w_512:
6091 case Intrinsic::x86_avx512_psrai_d_512:
6092 case Intrinsic::x86_avx512_psrai_q_512:
6093 case Intrinsic::x86_avx512_psra_q_256:
6094 case Intrinsic::x86_avx512_psra_q_128:
6095 case Intrinsic::x86_avx512_psrai_q_256:
6096 case Intrinsic::x86_avx512_psrai_q_128:
6097 case Intrinsic::x86_avx2_psll_w:
6098 case Intrinsic::x86_avx2_psll_d:
6099 case Intrinsic::x86_avx2_psll_q:
6100 case Intrinsic::x86_avx2_pslli_w:
6101 case Intrinsic::x86_avx2_pslli_d:
6102 case Intrinsic::x86_avx2_pslli_q:
6103 case Intrinsic::x86_avx2_psrl_w:
6104 case Intrinsic::x86_avx2_psrl_d:
6105 case Intrinsic::x86_avx2_psrl_q:
6106 case Intrinsic::x86_avx2_psra_w:
6107 case Intrinsic::x86_avx2_psra_d:
6108 case Intrinsic::x86_avx2_psrli_w:
6109 case Intrinsic::x86_avx2_psrli_d:
6110 case Intrinsic::x86_avx2_psrli_q:
6111 case Intrinsic::x86_avx2_psrai_w:
6112 case Intrinsic::x86_avx2_psrai_d:
6113 case Intrinsic::x86_sse2_psll_w:
6114 case Intrinsic::x86_sse2_psll_d:
6115 case Intrinsic::x86_sse2_psll_q:
6116 case Intrinsic::x86_sse2_pslli_w:
6117 case Intrinsic::x86_sse2_pslli_d:
6118 case Intrinsic::x86_sse2_pslli_q:
6119 case Intrinsic::x86_sse2_psrl_w:
6120 case Intrinsic::x86_sse2_psrl_d:
6121 case Intrinsic::x86_sse2_psrl_q:
6122 case Intrinsic::x86_sse2_psra_w:
6123 case Intrinsic::x86_sse2_psra_d:
6124 case Intrinsic::x86_sse2_psrli_w:
6125 case Intrinsic::x86_sse2_psrli_d:
6126 case Intrinsic::x86_sse2_psrli_q:
6127 case Intrinsic::x86_sse2_psrai_w:
6128 case Intrinsic::x86_sse2_psrai_d:
6129 case Intrinsic::x86_mmx_psll_w:
6130 case Intrinsic::x86_mmx_psll_d:
6131 case Intrinsic::x86_mmx_psll_q:
6132 case Intrinsic::x86_mmx_pslli_w:
6133 case Intrinsic::x86_mmx_pslli_d:
6134 case Intrinsic::x86_mmx_pslli_q:
6135 case Intrinsic::x86_mmx_psrl_w:
6136 case Intrinsic::x86_mmx_psrl_d:
6137 case Intrinsic::x86_mmx_psrl_q:
6138 case Intrinsic::x86_mmx_psra_w:
6139 case Intrinsic::x86_mmx_psra_d:
6140 case Intrinsic::x86_mmx_psrli_w:
6141 case Intrinsic::x86_mmx_psrli_d:
6142 case Intrinsic::x86_mmx_psrli_q:
6143 case Intrinsic::x86_mmx_psrai_w:
6144 case Intrinsic::x86_mmx_psrai_d:
6145 handleVectorShiftIntrinsic(I, /* Variable */ false);
6146 break;
6147 case Intrinsic::x86_avx2_psllv_d:
6148 case Intrinsic::x86_avx2_psllv_d_256:
6149 case Intrinsic::x86_avx512_psllv_d_512:
6150 case Intrinsic::x86_avx2_psllv_q:
6151 case Intrinsic::x86_avx2_psllv_q_256:
6152 case Intrinsic::x86_avx512_psllv_q_512:
6153 case Intrinsic::x86_avx2_psrlv_d:
6154 case Intrinsic::x86_avx2_psrlv_d_256:
6155 case Intrinsic::x86_avx512_psrlv_d_512:
6156 case Intrinsic::x86_avx2_psrlv_q:
6157 case Intrinsic::x86_avx2_psrlv_q_256:
6158 case Intrinsic::x86_avx512_psrlv_q_512:
6159 case Intrinsic::x86_avx2_psrav_d:
6160 case Intrinsic::x86_avx2_psrav_d_256:
6161 case Intrinsic::x86_avx512_psrav_d_512:
6162 case Intrinsic::x86_avx512_psrav_q_128:
6163 case Intrinsic::x86_avx512_psrav_q_256:
6164 case Intrinsic::x86_avx512_psrav_q_512:
6165 handleVectorShiftIntrinsic(I, /* Variable */ true);
6166 break;
6167
6168 // Pack with Signed/Unsigned Saturation
6169 case Intrinsic::x86_sse2_packsswb_128:
6170 case Intrinsic::x86_sse2_packssdw_128:
6171 case Intrinsic::x86_sse2_packuswb_128:
6172 case Intrinsic::x86_sse41_packusdw:
6173 case Intrinsic::x86_avx2_packsswb:
6174 case Intrinsic::x86_avx2_packssdw:
6175 case Intrinsic::x86_avx2_packuswb:
6176 case Intrinsic::x86_avx2_packusdw:
6177 // e.g., <64 x i8> @llvm.x86.avx512.packsswb.512
6178 // (<32 x i16> %a, <32 x i16> %b)
6179 // <32 x i16> @llvm.x86.avx512.packssdw.512
6180 // (<16 x i32> %a, <16 x i32> %b)
6181 // Note: AVX512 masked variants are auto-upgraded by LLVM.
6182 case Intrinsic::x86_avx512_packsswb_512:
6183 case Intrinsic::x86_avx512_packssdw_512:
6184 case Intrinsic::x86_avx512_packuswb_512:
6185 case Intrinsic::x86_avx512_packusdw_512:
6186 handleVectorPackIntrinsic(I);
6187 break;
6188
6189 case Intrinsic::x86_sse41_pblendvb:
6190 case Intrinsic::x86_sse41_blendvpd:
6191 case Intrinsic::x86_sse41_blendvps:
6192 case Intrinsic::x86_avx_blendv_pd_256:
6193 case Intrinsic::x86_avx_blendv_ps_256:
6194 case Intrinsic::x86_avx2_pblendvb:
6195 handleBlendvIntrinsic(I);
6196 break;
6197
6198 case Intrinsic::x86_avx_dp_ps_256:
6199 case Intrinsic::x86_sse41_dppd:
6200 case Intrinsic::x86_sse41_dpps:
6201 handleDppIntrinsic(I);
6202 break;
6203
6204 case Intrinsic::x86_mmx_packsswb:
6205 case Intrinsic::x86_mmx_packuswb:
6206 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 16);
6207 break;
6208
6209 case Intrinsic::x86_mmx_packssdw:
6210 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 32);
6211 break;
6212
6213 case Intrinsic::x86_mmx_psad_bw:
6214 handleVectorSadIntrinsic(I, IsMMX: true);
6215 break;
6216 case Intrinsic::x86_sse2_psad_bw:
6217 case Intrinsic::x86_avx2_psad_bw:
6218 handleVectorSadIntrinsic(I);
6219 break;
6220
6221 // Multiply and Add Packed Words
6222 // < 4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
6223 // < 8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
6224 // <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
6225 //
6226 // Multiply and Add Packed Signed and Unsigned Bytes
6227 // < 8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
6228 // <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
6229 // <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
6230 //
6231 // These intrinsics are auto-upgraded into non-masked forms:
6232 // < 4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128
6233 // (<8 x i16>, <8 x i16>, <4 x i32>, i8)
6234 // < 8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256
6235 // (<16 x i16>, <16 x i16>, <8 x i32>, i8)
6236 // <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512
6237 // (<32 x i16>, <32 x i16>, <16 x i32>, i16)
6238 // < 8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128
6239 // (<16 x i8>, <16 x i8>, <8 x i16>, i8)
6240 // <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256
6241 // (<32 x i8>, <32 x i8>, <16 x i16>, i16)
6242 // <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512
6243 // (<64 x i8>, <64 x i8>, <32 x i16>, i32)
6244 case Intrinsic::x86_sse2_pmadd_wd:
6245 case Intrinsic::x86_avx2_pmadd_wd:
6246 case Intrinsic::x86_avx512_pmaddw_d_512:
6247 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
6248 case Intrinsic::x86_avx2_pmadd_ub_sw:
6249 case Intrinsic::x86_avx512_pmaddubs_w_512:
6250 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6251 /*ZeroPurifies=*/true,
6252 /*EltSizeInBits=*/0,
6253 /*Lanes=*/kBothLanes);
6254 break;
6255
6256 // <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>)
6257 case Intrinsic::x86_ssse3_pmadd_ub_sw:
6258 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6259 /*ZeroPurifies=*/true,
6260 /*EltSizeInBits=*/8,
6261 /*Lanes=*/kBothLanes);
6262 break;
6263
6264 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>)
6265 case Intrinsic::x86_mmx_pmadd_wd:
6266 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6267 /*ZeroPurifies=*/true,
6268 /*EltSizeInBits=*/16,
6269 /*Lanes=*/kBothLanes);
6270 break;
6271
6272 // BFloat16 multiply-add to single-precision
6273 // <4 x float> llvm.aarch64.neon.bfmlalt
6274 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6275 case Intrinsic::aarch64_neon_bfmlalt:
6276 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6277 /*ZeroPurifies=*/false,
6278 /*EltSizeInBits=*/0,
6279 /*Lanes=*/kOddLanes);
6280 break;
6281
6282 // <4 x float> llvm.aarch64.neon.bfmlalb
6283 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6284 case Intrinsic::aarch64_neon_bfmlalb:
6285 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6286 /*ZeroPurifies=*/false,
6287 /*EltSizeInBits=*/0,
6288 /*Lanes=*/kEvenLanes);
6289 break;
6290
6291 // AVX Vector Neural Network Instructions: bytes
6292 //
6293 // Multiply and Add Signed Bytes
6294 // < 4 x i32> @llvm.x86.avx2.vpdpbssd.128
6295 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6296 // < 8 x i32> @llvm.x86.avx2.vpdpbssd.256
6297 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6298 // <16 x i32> @llvm.x86.avx10.vpdpbssd.512
6299 // (<16 x i32>, <64 x i8>, <64 x i8>)
6300 //
6301 // Multiply and Add Signed Bytes With Saturation
6302 // < 4 x i32> @llvm.x86.avx2.vpdpbssds.128
6303 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6304 // < 8 x i32> @llvm.x86.avx2.vpdpbssds.256
6305 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6306 // <16 x i32> @llvm.x86.avx10.vpdpbssds.512
6307 // (<16 x i32>, <64 x i8>, <64 x i8>)
6308 //
6309 // Multiply and Add Signed and Unsigned Bytes
6310 // < 4 x i32> @llvm.x86.avx2.vpdpbsud.128
6311 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6312 // < 8 x i32> @llvm.x86.avx2.vpdpbsud.256
6313 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6314 // <16 x i32> @llvm.x86.avx10.vpdpbsud.512
6315 // (<16 x i32>, <64 x i8>, <64 x i8>)
6316 //
6317 // Multiply and Add Signed and Unsigned Bytes With Saturation
6318 // < 4 x i32> @llvm.x86.avx2.vpdpbsuds.128
6319 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6320 // < 8 x i32> @llvm.x86.avx2.vpdpbsuds.256
6321 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6322 // <16 x i32> @llvm.x86.avx512.vpdpbusds.512
6323 // (<16 x i32>, <64 x i8>, <64 x i8>)
6324 //
6325 // Multiply and Add Unsigned and Signed Bytes
6326 // < 4 x i32> @llvm.x86.avx512.vpdpbusd.128
6327 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6328 // < 8 x i32> @llvm.x86.avx512.vpdpbusd.256
6329 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6330 // <16 x i32> @llvm.x86.avx512.vpdpbusd.512
6331 // (<16 x i32>, <64 x i8>, <64 x i8>)
6332 //
6333 // Multiply and Add Unsigned and Signed Bytes With Saturation
6334 // < 4 x i32> @llvm.x86.avx512.vpdpbusds.128
6335 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6336 // < 8 x i32> @llvm.x86.avx512.vpdpbusds.256
6337 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6338 // <16 x i32> @llvm.x86.avx10.vpdpbsuds.512
6339 // (<16 x i32>, <64 x i8>, <64 x i8>)
6340 //
6341 // Multiply and Add Unsigned Bytes
6342 // < 4 x i32> @llvm.x86.avx2.vpdpbuud.128
6343 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6344 // < 8 x i32> @llvm.x86.avx2.vpdpbuud.256
6345 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6346 // <16 x i32> @llvm.x86.avx10.vpdpbuud.512
6347 // (<16 x i32>, <64 x i8>, <64 x i8>)
6348 //
6349 // Multiply and Add Unsigned Bytes With Saturation
6350 // < 4 x i32> @llvm.x86.avx2.vpdpbuuds.128
6351 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6352 // < 8 x i32> @llvm.x86.avx2.vpdpbuuds.256
6353 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6354 // <16 x i32> @llvm.x86.avx10.vpdpbuuds.512
6355 // (<16 x i32>, <64 x i8>, <64 x i8>)
6356 //
6357 // These intrinsics are auto-upgraded into non-masked forms:
6358 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128
6359 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6360 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128
6361 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6362 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256
6363 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6364 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256
6365 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6366 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512
6367 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6368 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512
6369 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6370 //
6371 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128
6372 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6373 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128
6374 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6375 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256
6376 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6377 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256
6378 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6379 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512
6380 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6381 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512
6382 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6383 case Intrinsic::x86_avx512_vpdpbusd_128:
6384 case Intrinsic::x86_avx512_vpdpbusd_256:
6385 case Intrinsic::x86_avx512_vpdpbusd_512:
6386 case Intrinsic::x86_avx512_vpdpbusds_128:
6387 case Intrinsic::x86_avx512_vpdpbusds_256:
6388 case Intrinsic::x86_avx512_vpdpbusds_512:
6389 case Intrinsic::x86_avx2_vpdpbssd_128:
6390 case Intrinsic::x86_avx2_vpdpbssd_256:
6391 case Intrinsic::x86_avx10_vpdpbssd_512:
6392 case Intrinsic::x86_avx2_vpdpbssds_128:
6393 case Intrinsic::x86_avx2_vpdpbssds_256:
6394 case Intrinsic::x86_avx10_vpdpbssds_512:
6395 case Intrinsic::x86_avx2_vpdpbsud_128:
6396 case Intrinsic::x86_avx2_vpdpbsud_256:
6397 case Intrinsic::x86_avx10_vpdpbsud_512:
6398 case Intrinsic::x86_avx2_vpdpbsuds_128:
6399 case Intrinsic::x86_avx2_vpdpbsuds_256:
6400 case Intrinsic::x86_avx10_vpdpbsuds_512:
6401 case Intrinsic::x86_avx2_vpdpbuud_128:
6402 case Intrinsic::x86_avx2_vpdpbuud_256:
6403 case Intrinsic::x86_avx10_vpdpbuud_512:
6404 case Intrinsic::x86_avx2_vpdpbuuds_128:
6405 case Intrinsic::x86_avx2_vpdpbuuds_256:
6406 case Intrinsic::x86_avx10_vpdpbuuds_512:
6407 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
6408 /*ZeroPurifies=*/true,
6409 /*EltSizeInBits=*/0,
6410 /*Lanes=*/kBothLanes);
6411 break;
6412
6413 // AVX Vector Neural Network Instructions: words
6414 //
6415 // Multiply and Add Signed Word Integers
6416 // < 4 x i32> @llvm.x86.avx512.vpdpwssd.128
6417 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6418 // < 8 x i32> @llvm.x86.avx512.vpdpwssd.256
6419 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6420 // <16 x i32> @llvm.x86.avx512.vpdpwssd.512
6421 // (<16 x i32>, <32 x i16>, <32 x i16>)
6422 //
6423 // Multiply and Add Signed Word Integers With Saturation
6424 // < 4 x i32> @llvm.x86.avx512.vpdpwssds.128
6425 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6426 // < 8 x i32> @llvm.x86.avx512.vpdpwssds.256
6427 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6428 // <16 x i32> @llvm.x86.avx512.vpdpwssds.512
6429 // (<16 x i32>, <32 x i16>, <32 x i16>)
6430 //
6431 // Multiply and Add Signed and Unsigned Word Integers
6432 // < 4 x i32> @llvm.x86.avx2.vpdpwsud.128
6433 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6434 // < 8 x i32> @llvm.x86.avx2.vpdpwsud.256
6435 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6436 // <16 x i32> @llvm.x86.avx10.vpdpwsud.512
6437 // (<16 x i32>, <32 x i16>, <32 x i16>)
6438 //
6439 // Multiply and Add Signed and Unsigned Word Integers With Saturation
6440 // < 4 x i32> @llvm.x86.avx2.vpdpwsuds.128
6441 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6442 // < 8 x i32> @llvm.x86.avx2.vpdpwsuds.256
6443 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6444 // <16 x i32> @llvm.x86.avx10.vpdpwsuds.512
6445 // (<16 x i32>, <32 x i16>, <32 x i16>)
6446 //
6447 // Multiply and Add Unsigned and Signed Word Integers
6448 // < 4 x i32> @llvm.x86.avx2.vpdpwusd.128
6449 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6450 // < 8 x i32> @llvm.x86.avx2.vpdpwusd.256
6451 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6452 // <16 x i32> @llvm.x86.avx10.vpdpwusd.512
6453 // (<16 x i32>, <32 x i16>, <32 x i16>)
6454 //
6455 // Multiply and Add Unsigned and Signed Word Integers With Saturation
6456 // < 4 x i32> @llvm.x86.avx2.vpdpwusds.128
6457 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6458 // < 8 x i32> @llvm.x86.avx2.vpdpwusds.256
6459 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6460 // <16 x i32> @llvm.x86.avx10.vpdpwusds.512
6461 // (<16 x i32>, <32 x i16>, <32 x i16>)
6462 //
6463 // Multiply and Add Unsigned and Unsigned Word Integers
6464 // < 4 x i32> @llvm.x86.avx2.vpdpwuud.128
6465 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6466 // < 8 x i32> @llvm.x86.avx2.vpdpwuud.256
6467 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6468 // <16 x i32> @llvm.x86.avx10.vpdpwuud.512
6469 // (<16 x i32>, <32 x i16>, <32 x i16>)
6470 //
6471 // Multiply and Add Unsigned and Unsigned Word Integers With Saturation
6472 // < 4 x i32> @llvm.x86.avx2.vpdpwuuds.128
6473 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6474 // < 8 x i32> @llvm.x86.avx2.vpdpwuuds.256
6475 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6476 // <16 x i32> @llvm.x86.avx10.vpdpwuuds.512
6477 // (<16 x i32>, <32 x i16>, <32 x i16>)
6478 //
6479 // These intrinsics are auto-upgraded into non-masked forms:
6480 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128
6481 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6482 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128
6483 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6484 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256
6485 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6486 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256
6487 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6488 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512
6489 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6490 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512
6491 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6492 //
6493 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128
6494 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6495 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128
6496 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6497 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256
6498 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6499 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256
6500 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6501 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512
6502 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6503 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512
6504 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6505 case Intrinsic::x86_avx512_vpdpwssd_128:
6506 case Intrinsic::x86_avx512_vpdpwssd_256:
6507 case Intrinsic::x86_avx512_vpdpwssd_512:
6508 case Intrinsic::x86_avx512_vpdpwssds_128:
6509 case Intrinsic::x86_avx512_vpdpwssds_256:
6510 case Intrinsic::x86_avx512_vpdpwssds_512:
6511 case Intrinsic::x86_avx2_vpdpwsud_128:
6512 case Intrinsic::x86_avx2_vpdpwsud_256:
6513 case Intrinsic::x86_avx10_vpdpwsud_512:
6514 case Intrinsic::x86_avx2_vpdpwsuds_128:
6515 case Intrinsic::x86_avx2_vpdpwsuds_256:
6516 case Intrinsic::x86_avx10_vpdpwsuds_512:
6517 case Intrinsic::x86_avx2_vpdpwusd_128:
6518 case Intrinsic::x86_avx2_vpdpwusd_256:
6519 case Intrinsic::x86_avx10_vpdpwusd_512:
6520 case Intrinsic::x86_avx2_vpdpwusds_128:
6521 case Intrinsic::x86_avx2_vpdpwusds_256:
6522 case Intrinsic::x86_avx10_vpdpwusds_512:
6523 case Intrinsic::x86_avx2_vpdpwuud_128:
6524 case Intrinsic::x86_avx2_vpdpwuud_256:
6525 case Intrinsic::x86_avx10_vpdpwuud_512:
6526 case Intrinsic::x86_avx2_vpdpwuuds_128:
6527 case Intrinsic::x86_avx2_vpdpwuuds_256:
6528 case Intrinsic::x86_avx10_vpdpwuuds_512:
6529 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6530 /*ZeroPurifies=*/true,
6531 /*EltSizeInBits=*/0,
6532 /*Lanes=*/kBothLanes);
6533 break;
6534
6535 // Dot Product of BF16 Pairs Accumulated Into Packed Single
6536 // Precision
6537 // <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128
6538 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6539 // <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256
6540 // (<8 x float>, <16 x bfloat>, <16 x bfloat>)
6541 // <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512
6542 // (<16 x float>, <32 x bfloat>, <32 x bfloat>)
6543 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6544 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6545 case Intrinsic::x86_avx512bf16_dpbf16ps_512:
6546 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6547 /*ZeroPurifies=*/false,
6548 /*EltSizeInBits=*/0,
6549 /*Lanes=*/kBothLanes);
6550 break;
6551
6552 case Intrinsic::x86_sse_cmp_ss:
6553 case Intrinsic::x86_sse2_cmp_sd:
6554 case Intrinsic::x86_sse_comieq_ss:
6555 case Intrinsic::x86_sse_comilt_ss:
6556 case Intrinsic::x86_sse_comile_ss:
6557 case Intrinsic::x86_sse_comigt_ss:
6558 case Intrinsic::x86_sse_comige_ss:
6559 case Intrinsic::x86_sse_comineq_ss:
6560 case Intrinsic::x86_sse_ucomieq_ss:
6561 case Intrinsic::x86_sse_ucomilt_ss:
6562 case Intrinsic::x86_sse_ucomile_ss:
6563 case Intrinsic::x86_sse_ucomigt_ss:
6564 case Intrinsic::x86_sse_ucomige_ss:
6565 case Intrinsic::x86_sse_ucomineq_ss:
6566 case Intrinsic::x86_sse2_comieq_sd:
6567 case Intrinsic::x86_sse2_comilt_sd:
6568 case Intrinsic::x86_sse2_comile_sd:
6569 case Intrinsic::x86_sse2_comigt_sd:
6570 case Intrinsic::x86_sse2_comige_sd:
6571 case Intrinsic::x86_sse2_comineq_sd:
6572 case Intrinsic::x86_sse2_ucomieq_sd:
6573 case Intrinsic::x86_sse2_ucomilt_sd:
6574 case Intrinsic::x86_sse2_ucomile_sd:
6575 case Intrinsic::x86_sse2_ucomigt_sd:
6576 case Intrinsic::x86_sse2_ucomige_sd:
6577 case Intrinsic::x86_sse2_ucomineq_sd:
6578 handleVectorCompareScalarIntrinsic(I);
6579 break;
6580
6581 case Intrinsic::x86_avx_cmp_pd_256:
6582 case Intrinsic::x86_avx_cmp_ps_256:
6583 case Intrinsic::x86_sse2_cmp_pd:
6584 case Intrinsic::x86_sse_cmp_ps:
6585 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/true);
6586 break;
6587
6588 case Intrinsic::x86_bmi_bextr_32:
6589 case Intrinsic::x86_bmi_bextr_64:
6590 case Intrinsic::x86_bmi_bzhi_32:
6591 case Intrinsic::x86_bmi_bzhi_64:
6592 handleGenericBitManipulation(I);
6593 break;
6594
6595 case Intrinsic::x86_pclmulqdq:
6596 case Intrinsic::x86_pclmulqdq_256:
6597 case Intrinsic::x86_pclmulqdq_512:
6598 handlePclmulIntrinsic(I);
6599 break;
6600
6601 case Intrinsic::x86_avx_round_pd_256:
6602 case Intrinsic::x86_avx_round_ps_256:
6603 case Intrinsic::x86_sse41_round_pd:
6604 case Intrinsic::x86_sse41_round_ps:
6605 handleRoundPdPsIntrinsic(I);
6606 break;
6607
6608 case Intrinsic::x86_sse41_round_sd:
6609 case Intrinsic::x86_sse41_round_ss:
6610 handleUnarySdSsIntrinsic(I);
6611 break;
6612
6613 case Intrinsic::x86_sse2_max_sd:
6614 case Intrinsic::x86_sse_max_ss:
6615 case Intrinsic::x86_sse2_min_sd:
6616 case Intrinsic::x86_sse_min_ss:
6617 handleBinarySdSsIntrinsic(I);
6618 break;
6619
6620 case Intrinsic::x86_avx_vtestc_pd:
6621 case Intrinsic::x86_avx_vtestc_pd_256:
6622 case Intrinsic::x86_avx_vtestc_ps:
6623 case Intrinsic::x86_avx_vtestc_ps_256:
6624 case Intrinsic::x86_avx_vtestnzc_pd:
6625 case Intrinsic::x86_avx_vtestnzc_pd_256:
6626 case Intrinsic::x86_avx_vtestnzc_ps:
6627 case Intrinsic::x86_avx_vtestnzc_ps_256:
6628 case Intrinsic::x86_avx_vtestz_pd:
6629 case Intrinsic::x86_avx_vtestz_pd_256:
6630 case Intrinsic::x86_avx_vtestz_ps:
6631 case Intrinsic::x86_avx_vtestz_ps_256:
6632 case Intrinsic::x86_avx_ptestc_256:
6633 case Intrinsic::x86_avx_ptestnzc_256:
6634 case Intrinsic::x86_avx_ptestz_256:
6635 case Intrinsic::x86_sse41_ptestc:
6636 case Intrinsic::x86_sse41_ptestnzc:
6637 case Intrinsic::x86_sse41_ptestz:
6638 handleVtestIntrinsic(I);
6639 break;
6640
6641 // Packed Horizontal Add/Subtract
6642 case Intrinsic::x86_ssse3_phadd_w:
6643 case Intrinsic::x86_ssse3_phadd_w_128:
6644 case Intrinsic::x86_ssse3_phsub_w:
6645 case Intrinsic::x86_ssse3_phsub_w_128:
6646 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6647 /*ReinterpretElemWidth=*/16);
6648 break;
6649
6650 case Intrinsic::x86_avx2_phadd_w:
6651 case Intrinsic::x86_avx2_phsub_w:
6652 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6653 /*ReinterpretElemWidth=*/16);
6654 break;
6655
6656 // Packed Horizontal Add/Subtract
6657 case Intrinsic::x86_ssse3_phadd_d:
6658 case Intrinsic::x86_ssse3_phadd_d_128:
6659 case Intrinsic::x86_ssse3_phsub_d:
6660 case Intrinsic::x86_ssse3_phsub_d_128:
6661 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6662 /*ReinterpretElemWidth=*/32);
6663 break;
6664
6665 case Intrinsic::x86_avx2_phadd_d:
6666 case Intrinsic::x86_avx2_phsub_d:
6667 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6668 /*ReinterpretElemWidth=*/32);
6669 break;
6670
6671 // Packed Horizontal Add/Subtract and Saturate
6672 case Intrinsic::x86_ssse3_phadd_sw:
6673 case Intrinsic::x86_ssse3_phadd_sw_128:
6674 case Intrinsic::x86_ssse3_phsub_sw:
6675 case Intrinsic::x86_ssse3_phsub_sw_128:
6676 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6677 /*ReinterpretElemWidth=*/16);
6678 break;
6679
6680 case Intrinsic::x86_avx2_phadd_sw:
6681 case Intrinsic::x86_avx2_phsub_sw:
6682 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6683 /*ReinterpretElemWidth=*/16);
6684 break;
6685
6686 // Packed Single/Double Precision Floating-Point Horizontal Add
6687 case Intrinsic::x86_sse3_hadd_ps:
6688 case Intrinsic::x86_sse3_hadd_pd:
6689 case Intrinsic::x86_sse3_hsub_ps:
6690 case Intrinsic::x86_sse3_hsub_pd:
6691 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
6692 break;
6693
6694 case Intrinsic::x86_avx_hadd_pd_256:
6695 case Intrinsic::x86_avx_hadd_ps_256:
6696 case Intrinsic::x86_avx_hsub_pd_256:
6697 case Intrinsic::x86_avx_hsub_ps_256:
6698 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2);
6699 break;
6700
6701 case Intrinsic::x86_avx_maskstore_ps:
6702 case Intrinsic::x86_avx_maskstore_pd:
6703 case Intrinsic::x86_avx_maskstore_ps_256:
6704 case Intrinsic::x86_avx_maskstore_pd_256:
6705 case Intrinsic::x86_avx2_maskstore_d:
6706 case Intrinsic::x86_avx2_maskstore_q:
6707 case Intrinsic::x86_avx2_maskstore_d_256:
6708 case Intrinsic::x86_avx2_maskstore_q_256: {
6709 handleAVXMaskedStore(I);
6710 break;
6711 }
6712
6713 case Intrinsic::x86_avx_maskload_ps:
6714 case Intrinsic::x86_avx_maskload_pd:
6715 case Intrinsic::x86_avx_maskload_ps_256:
6716 case Intrinsic::x86_avx_maskload_pd_256:
6717 case Intrinsic::x86_avx2_maskload_d:
6718 case Intrinsic::x86_avx2_maskload_q:
6719 case Intrinsic::x86_avx2_maskload_d_256:
6720 case Intrinsic::x86_avx2_maskload_q_256: {
6721 handleAVXMaskedLoad(I);
6722 break;
6723 }
6724
6725 // Packed
6726 case Intrinsic::x86_avx512fp16_add_ph_512:
6727 case Intrinsic::x86_avx512fp16_sub_ph_512:
6728 case Intrinsic::x86_avx512fp16_mul_ph_512:
6729 case Intrinsic::x86_avx512fp16_div_ph_512:
6730 case Intrinsic::x86_avx512fp16_max_ph_512:
6731 case Intrinsic::x86_avx512fp16_min_ph_512:
6732 case Intrinsic::x86_avx512_min_ps_512:
6733 case Intrinsic::x86_avx512_min_pd_512:
6734 case Intrinsic::x86_avx512_max_ps_512:
6735 case Intrinsic::x86_avx512_max_pd_512: {
6736 // These AVX512 variants contain the rounding mode as a trailing flag.
6737 // Earlier variants do not have a trailing flag and are already handled
6738 // by maybeHandleSimpleNomemIntrinsic(I, 0) via
6739 // maybeHandleUnknownIntrinsic.
6740 [[maybe_unused]] bool Success =
6741 maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/1);
6742 assert(Success);
6743 break;
6744 }
6745
6746 case Intrinsic::x86_avx_vpermilvar_pd:
6747 case Intrinsic::x86_avx_vpermilvar_pd_256:
6748 case Intrinsic::x86_avx512_vpermilvar_pd_512:
6749 case Intrinsic::x86_avx_vpermilvar_ps:
6750 case Intrinsic::x86_avx_vpermilvar_ps_256:
6751 case Intrinsic::x86_avx512_vpermilvar_ps_512: {
6752 handleAVXVpermilvar(I);
6753 break;
6754 }
6755
6756 case Intrinsic::x86_avx512_vpermi2var_d_128:
6757 case Intrinsic::x86_avx512_vpermi2var_d_256:
6758 case Intrinsic::x86_avx512_vpermi2var_d_512:
6759 case Intrinsic::x86_avx512_vpermi2var_hi_128:
6760 case Intrinsic::x86_avx512_vpermi2var_hi_256:
6761 case Intrinsic::x86_avx512_vpermi2var_hi_512:
6762 case Intrinsic::x86_avx512_vpermi2var_pd_128:
6763 case Intrinsic::x86_avx512_vpermi2var_pd_256:
6764 case Intrinsic::x86_avx512_vpermi2var_pd_512:
6765 case Intrinsic::x86_avx512_vpermi2var_ps_128:
6766 case Intrinsic::x86_avx512_vpermi2var_ps_256:
6767 case Intrinsic::x86_avx512_vpermi2var_ps_512:
6768 case Intrinsic::x86_avx512_vpermi2var_q_128:
6769 case Intrinsic::x86_avx512_vpermi2var_q_256:
6770 case Intrinsic::x86_avx512_vpermi2var_q_512:
6771 case Intrinsic::x86_avx512_vpermi2var_qi_128:
6772 case Intrinsic::x86_avx512_vpermi2var_qi_256:
6773 case Intrinsic::x86_avx512_vpermi2var_qi_512:
6774 handleAVXVpermi2var(I);
6775 break;
6776
6777 // Packed Shuffle
6778 // llvm.x86.sse.pshuf.w(<1 x i64>, i8)
6779 // llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>)
6780 // llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
6781 // llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
6782 // llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
6783 //
6784 // The following intrinsics are auto-upgraded:
6785 // llvm.x86.sse2.pshuf.d(<4 x i32>, i8)
6786 // llvm.x86.sse2.gpshufh.w(<8 x i16>, i8)
6787 // llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
6788 case Intrinsic::x86_avx2_pshuf_b:
6789 case Intrinsic::x86_sse_pshuf_w:
6790 case Intrinsic::x86_ssse3_pshuf_b_128:
6791 case Intrinsic::x86_ssse3_pshuf_b:
6792 case Intrinsic::x86_avx512_pshuf_b_512:
6793 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6794 /*trailingVerbatimArgs=*/1,
6795 /*forceIntegerIntrinsic=*/false);
6796 break;
6797
6798 // AVX512 PMOV: Packed MOV, with truncation
6799 // Precisely handled by applying the same intrinsic to the shadow
6800 case Intrinsic::x86_avx512_mask_pmov_dw_128:
6801 case Intrinsic::x86_avx512_mask_pmov_db_128:
6802 case Intrinsic::x86_avx512_mask_pmov_qb_128:
6803 case Intrinsic::x86_avx512_mask_pmov_qw_128:
6804 case Intrinsic::x86_avx512_mask_pmov_qd_128:
6805 case Intrinsic::x86_avx512_mask_pmov_wb_128:
6806 case Intrinsic::x86_avx512_mask_pmov_dw_256:
6807 case Intrinsic::x86_avx512_mask_pmov_db_256:
6808 case Intrinsic::x86_avx512_mask_pmov_qb_256:
6809 case Intrinsic::x86_avx512_mask_pmov_qw_256:
6810 case Intrinsic::x86_avx512_mask_pmov_dw_512:
6811 case Intrinsic::x86_avx512_mask_pmov_db_512:
6812 case Intrinsic::x86_avx512_mask_pmov_qb_512:
6813 case Intrinsic::x86_avx512_mask_pmov_qw_512: {
6814 // Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} were removed in
6815 // f608dc1f5775ee880e8ea30e2d06ab5a4a935c22
6816 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6817 /*trailingVerbatimArgs=*/1,
6818 /*forceIntegerIntrinsic=*/false);
6819 break;
6820 }
6821
6822 // AVX512 PMOV{S,US}: Packed MOV, with signed/unsigned saturation
6823 // Approximately handled using the corresponding truncation intrinsic
6824 // TODO: improve handleAVX512VectorDownConvert to precisely model saturation
6825 case Intrinsic::x86_avx512_mask_pmovs_dw_512:
6826 case Intrinsic::x86_avx512_mask_pmovus_dw_512: {
6827 handleIntrinsicByApplyingToShadow(
6828 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_512,
6829 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6830 break;
6831 }
6832
6833 case Intrinsic::x86_avx512_mask_pmovs_dw_256:
6834 case Intrinsic::x86_avx512_mask_pmovus_dw_256:
6835 handleIntrinsicByApplyingToShadow(
6836 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_256,
6837 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6838 break;
6839
6840 case Intrinsic::x86_avx512_mask_pmovs_dw_128:
6841 case Intrinsic::x86_avx512_mask_pmovus_dw_128:
6842 handleIntrinsicByApplyingToShadow(
6843 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_128,
6844 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6845 break;
6846
6847 case Intrinsic::x86_avx512_mask_pmovs_db_512:
6848 case Intrinsic::x86_avx512_mask_pmovus_db_512: {
6849 handleIntrinsicByApplyingToShadow(
6850 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_512,
6851 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6852 break;
6853 }
6854
6855 case Intrinsic::x86_avx512_mask_pmovs_db_256:
6856 case Intrinsic::x86_avx512_mask_pmovus_db_256:
6857 handleIntrinsicByApplyingToShadow(
6858 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_256,
6859 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6860 break;
6861
6862 case Intrinsic::x86_avx512_mask_pmovs_db_128:
6863 case Intrinsic::x86_avx512_mask_pmovus_db_128:
6864 handleIntrinsicByApplyingToShadow(
6865 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_128,
6866 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6867 break;
6868
6869 case Intrinsic::x86_avx512_mask_pmovs_qb_512:
6870 case Intrinsic::x86_avx512_mask_pmovus_qb_512: {
6871 handleIntrinsicByApplyingToShadow(
6872 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_512,
6873 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6874 break;
6875 }
6876
6877 case Intrinsic::x86_avx512_mask_pmovs_qb_256:
6878 case Intrinsic::x86_avx512_mask_pmovus_qb_256:
6879 handleIntrinsicByApplyingToShadow(
6880 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_256,
6881 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6882 break;
6883
6884 case Intrinsic::x86_avx512_mask_pmovs_qb_128:
6885 case Intrinsic::x86_avx512_mask_pmovus_qb_128:
6886 handleIntrinsicByApplyingToShadow(
6887 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_128,
6888 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6889 break;
6890
6891 case Intrinsic::x86_avx512_mask_pmovs_qw_512:
6892 case Intrinsic::x86_avx512_mask_pmovus_qw_512: {
6893 handleIntrinsicByApplyingToShadow(
6894 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_512,
6895 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6896 break;
6897 }
6898
6899 case Intrinsic::x86_avx512_mask_pmovs_qw_256:
6900 case Intrinsic::x86_avx512_mask_pmovus_qw_256:
6901 handleIntrinsicByApplyingToShadow(
6902 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_256,
6903 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6904 break;
6905
6906 case Intrinsic::x86_avx512_mask_pmovs_qw_128:
6907 case Intrinsic::x86_avx512_mask_pmovus_qw_128:
6908 handleIntrinsicByApplyingToShadow(
6909 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_128,
6910 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6911 break;
6912
6913 case Intrinsic::x86_avx512_mask_pmovs_qd_128:
6914 case Intrinsic::x86_avx512_mask_pmovus_qd_128:
6915 handleIntrinsicByApplyingToShadow(
6916 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qd_128,
6917 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6918 break;
6919
6920 case Intrinsic::x86_avx512_mask_pmovs_wb_128:
6921 case Intrinsic::x86_avx512_mask_pmovus_wb_128:
6922 handleIntrinsicByApplyingToShadow(
6923 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_wb_128,
6924 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6925 break;
6926
6927 case Intrinsic::x86_avx512_mask_pmovs_qd_256:
6928 case Intrinsic::x86_avx512_mask_pmovus_qd_256:
6929 case Intrinsic::x86_avx512_mask_pmovs_wb_256:
6930 case Intrinsic::x86_avx512_mask_pmovus_wb_256:
6931 case Intrinsic::x86_avx512_mask_pmovs_qd_512:
6932 case Intrinsic::x86_avx512_mask_pmovus_qd_512:
6933 case Intrinsic::x86_avx512_mask_pmovs_wb_512:
6934 case Intrinsic::x86_avx512_mask_pmovus_wb_512: {
6935 // Since Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} do not exist,
6936 // we cannot use handleIntrinsicByApplyingToShadow. Instead, we call the
6937 // slow-path handler.
6938 handleAVX512VectorDownConvert(I);
6939 break;
6940 }
6941
6942 // e.g.,
6943 // <16 x float> @llvm.x86.avx512.mask.compress
6944 // (<16 x float> %data, <16 x float> %passthru,
6945 // <16 x i1> %mask)
6946 // <16 x i32> @llvm.x86.avx512.mask.compress
6947 // (<16 x i32> %data, <16 x i32> %passthru,
6948 // <16 x i1> %mask)
6949 case Intrinsic::x86_avx512_mask_compress:
6950 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6951 /*trailingVerbatimArgs=*/1,
6952 /*forceIntegerIntrinsic=*/true);
6953 break;
6954
6955 // AVX512/AVX10 Reciprocal
6956 // <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
6957 // (<16 x float>, <16 x float>, i16)
6958 // <8 x float> @llvm.x86.avx512.rsqrt14.ps.256
6959 // (<8 x float>, <8 x float>, i8)
6960 // <4 x float> @llvm.x86.avx512.rsqrt14.ps.128
6961 // (<4 x float>, <4 x float>, i8)
6962 //
6963 // <8 x double> @llvm.x86.avx512.rsqrt14.pd.512
6964 // (<8 x double>, <8 x double>, i8)
6965 // <4 x double> @llvm.x86.avx512.rsqrt14.pd.256
6966 // (<4 x double>, <4 x double>, i8)
6967 // <2 x double> @llvm.x86.avx512.rsqrt14.pd.128
6968 // (<2 x double>, <2 x double>, i8)
6969 //
6970 // <32 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.512
6971 // (<32 x bfloat>, <32 x bfloat>, i32)
6972 // <16 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.256
6973 // (<16 x bfloat>, <16 x bfloat>, i16)
6974 // <8 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.128
6975 // (<8 x bfloat>, <8 x bfloat>, i8)
6976 //
6977 // <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512
6978 // (<32 x half>, <32 x half>, i32)
6979 // <16 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.256
6980 // (<16 x half>, <16 x half>, i16)
6981 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.128
6982 // (<8 x half>, <8 x half>, i8)
6983 //
6984 // TODO: 3-operand variants are not handled:
6985 // <2 x double> @llvm.x86.avx512.rsqrt14.sd
6986 // (<2 x double>, <2 x double>, <2 x double>, i8)
6987 // <4 x float> @llvm.x86.avx512.rsqrt14.ss
6988 // (<4 x float>, <4 x float>, <4 x float>, i8)
6989 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh
6990 // (<8 x half>, <8 x half>, <8 x half>, i8)
6991 case Intrinsic::x86_avx512_rsqrt14_ps_512:
6992 case Intrinsic::x86_avx512_rsqrt14_ps_256:
6993 case Intrinsic::x86_avx512_rsqrt14_ps_128:
6994 case Intrinsic::x86_avx512_rsqrt14_pd_512:
6995 case Intrinsic::x86_avx512_rsqrt14_pd_256:
6996 case Intrinsic::x86_avx512_rsqrt14_pd_128:
6997 case Intrinsic::x86_avx10_mask_rsqrt_bf16_512:
6998 case Intrinsic::x86_avx10_mask_rsqrt_bf16_256:
6999 case Intrinsic::x86_avx10_mask_rsqrt_bf16_128:
7000 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_512:
7001 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_256:
7002 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_128:
7003 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7004 /*WriteThruIndex=*/1,
7005 /*MaskIndex=*/2);
7006 break;
7007
7008 // AVX512/AVX10 Reciprocal Square Root
7009 // <16 x float> @llvm.x86.avx512.rcp14.ps.512
7010 // (<16 x float>, <16 x float>, i16)
7011 // <8 x float> @llvm.x86.avx512.rcp14.ps.256
7012 // (<8 x float>, <8 x float>, i8)
7013 // <4 x float> @llvm.x86.avx512.rcp14.ps.128
7014 // (<4 x float>, <4 x float>, i8)
7015 //
7016 // <8 x double> @llvm.x86.avx512.rcp14.pd.512
7017 // (<8 x double>, <8 x double>, i8)
7018 // <4 x double> @llvm.x86.avx512.rcp14.pd.256
7019 // (<4 x double>, <4 x double>, i8)
7020 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
7021 // (<2 x double>, <2 x double>, i8)
7022 //
7023 // <32 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.512
7024 // (<32 x bfloat>, <32 x bfloat>, i32)
7025 // <16 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.256
7026 // (<16 x bfloat>, <16 x bfloat>, i16)
7027 // <8 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.128
7028 // (<8 x bfloat>, <8 x bfloat>, i8)
7029 //
7030 // <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512
7031 // (<32 x half>, <32 x half>, i32)
7032 // <16 x half> @llvm.x86.avx512fp16.mask.rcp.ph.256
7033 // (<16 x half>, <16 x half>, i16)
7034 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.ph.128
7035 // (<8 x half>, <8 x half>, i8)
7036 //
7037 // TODO: 3-operand variants are not handled:
7038 // <2 x double> @llvm.x86.avx512.rcp14.sd
7039 // (<2 x double>, <2 x double>, <2 x double>, i8)
7040 // <4 x float> @llvm.x86.avx512.rcp14.ss
7041 // (<4 x float>, <4 x float>, <4 x float>, i8)
7042 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh
7043 // (<8 x half>, <8 x half>, <8 x half>, i8)
7044 case Intrinsic::x86_avx512_rcp14_ps_512:
7045 case Intrinsic::x86_avx512_rcp14_ps_256:
7046 case Intrinsic::x86_avx512_rcp14_ps_128:
7047 case Intrinsic::x86_avx512_rcp14_pd_512:
7048 case Intrinsic::x86_avx512_rcp14_pd_256:
7049 case Intrinsic::x86_avx512_rcp14_pd_128:
7050 case Intrinsic::x86_avx10_mask_rcp_bf16_512:
7051 case Intrinsic::x86_avx10_mask_rcp_bf16_256:
7052 case Intrinsic::x86_avx10_mask_rcp_bf16_128:
7053 case Intrinsic::x86_avx512fp16_mask_rcp_ph_512:
7054 case Intrinsic::x86_avx512fp16_mask_rcp_ph_256:
7055 case Intrinsic::x86_avx512fp16_mask_rcp_ph_128:
7056 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7057 /*WriteThruIndex=*/1,
7058 /*MaskIndex=*/2);
7059 break;
7060
7061 // <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512
7062 // (<32 x half>, i32, <32 x half>, i32, i32)
7063 // <16 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.256
7064 // (<16 x half>, i32, <16 x half>, i32, i16)
7065 // <8 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.128
7066 // (<8 x half>, i32, <8 x half>, i32, i8)
7067 //
7068 // <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512
7069 // (<16 x float>, i32, <16 x float>, i16, i32)
7070 // <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256
7071 // (<8 x float>, i32, <8 x float>, i8)
7072 // <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128
7073 // (<4 x float>, i32, <4 x float>, i8)
7074 //
7075 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
7076 // (<8 x double>, i32, <8 x double>, i8, i32)
7077 // A Imm WriteThru Mask Rounding
7078 // <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256
7079 // (<4 x double>, i32, <4 x double>, i8)
7080 // <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128
7081 // (<2 x double>, i32, <2 x double>, i8)
7082 // A Imm WriteThru Mask
7083 //
7084 // <32 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.512
7085 // (<32 x bfloat>, i32, <32 x bfloat>, i32)
7086 // <16 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.256
7087 // (<16 x bfloat>, i32, <16 x bfloat>, i16)
7088 // <8 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.128
7089 // (<8 x bfloat>, i32, <8 x bfloat>, i8)
7090 //
7091 // Not supported: three vectors
7092 // - <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh
7093 // (<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
7094 // - <4 x float> @llvm.x86.avx512.mask.rndscale.ss
7095 // (<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
7096 // - <2 x double> @llvm.x86.avx512.mask.rndscale.sd
7097 // (<2 x double>, <2 x double>, <2 x double>, i8, i32,
7098 // i32)
7099 // A B WriteThru Mask Imm
7100 // Rounding
7101 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_512:
7102 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_256:
7103 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_128:
7104 case Intrinsic::x86_avx512_mask_rndscale_ps_512:
7105 case Intrinsic::x86_avx512_mask_rndscale_ps_256:
7106 case Intrinsic::x86_avx512_mask_rndscale_ps_128:
7107 case Intrinsic::x86_avx512_mask_rndscale_pd_512:
7108 case Intrinsic::x86_avx512_mask_rndscale_pd_256:
7109 case Intrinsic::x86_avx512_mask_rndscale_pd_128:
7110 case Intrinsic::x86_avx10_mask_rndscale_bf16_512:
7111 case Intrinsic::x86_avx10_mask_rndscale_bf16_256:
7112 case Intrinsic::x86_avx10_mask_rndscale_bf16_128:
7113 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7114 /*WriteThruIndex=*/2,
7115 /*MaskIndex=*/3);
7116 break;
7117
7118 // AVX512 Vector Scale Float* Packed
7119 //
7120 // < 8 x double> @llvm.x86.avx512.mask.scalef.pd.512
7121 // (<8 x double>, <8 x double>, <8 x double>, i8, i32)
7122 // A B WriteThru Msk Round
7123 // < 4 x double> @llvm.x86.avx512.mask.scalef.pd.256
7124 // (<4 x double>, <4 x double>, <4 x double>, i8)
7125 // < 2 x double> @llvm.x86.avx512.mask.scalef.pd.128
7126 // (<2 x double>, <2 x double>, <2 x double>, i8)
7127 //
7128 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
7129 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
7130 // < 8 x float> @llvm.x86.avx512.mask.scalef.ps.256
7131 // (<8 x float>, <8 x float>, <8 x float>, i8)
7132 // < 4 x float> @llvm.x86.avx512.mask.scalef.ps.128
7133 // (<4 x float>, <4 x float>, <4 x float>, i8)
7134 //
7135 // <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512
7136 // (<32 x half>, <32 x half>, <32 x half>, i32, i32)
7137 // <16 x half> @llvm.x86.avx512fp16.mask.scalef.ph.256
7138 // (<16 x half>, <16 x half>, <16 x half>, i16)
7139 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.ph.128
7140 // (<8 x half>, <8 x half>, <8 x half>, i8)
7141 //
7142 // TODO: AVX10
7143 // <32 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.512
7144 // (<32 x bfloat>, <32 x bfloat>, <32 x bfloat>, i32)
7145 // <16 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.256
7146 // (<16 x bfloat>, <16 x bfloat>, <16 x bfloat>, i16)
7147 // < 8 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.128
7148 // (<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i8)
7149 case Intrinsic::x86_avx512_mask_scalef_pd_512:
7150 case Intrinsic::x86_avx512_mask_scalef_pd_256:
7151 case Intrinsic::x86_avx512_mask_scalef_pd_128:
7152 case Intrinsic::x86_avx512_mask_scalef_ps_512:
7153 case Intrinsic::x86_avx512_mask_scalef_ps_256:
7154 case Intrinsic::x86_avx512_mask_scalef_ps_128:
7155 case Intrinsic::x86_avx512fp16_mask_scalef_ph_512:
7156 case Intrinsic::x86_avx512fp16_mask_scalef_ph_256:
7157 case Intrinsic::x86_avx512fp16_mask_scalef_ph_128:
7158 // The AVX512 512-bit operand variants have an extra operand (the
7159 // Rounding mode). The extra operand, if present, will be
7160 // automatically checked by the handler.
7161 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0, 1},
7162 /*WriteThruIndex=*/2,
7163 /*MaskIndex=*/3);
7164 break;
7165
7166 // TODO: AVX512 Vector Scale Float* Scalar
7167 //
7168 // This is different from the Packed variant, because some bits are copied,
7169 // and some bits are zeroed.
7170 //
7171 // < 4 x float> @llvm.x86.avx512.mask.scalef.ss
7172 // (<4 x float>, <4 x float>, <4 x float>, i8, i32)
7173 //
7174 // < 2 x double> @llvm.x86.avx512.mask.scalef.sd
7175 // (<2 x double>, <2 x double>, <2 x double>, i8, i32)
7176 //
7177 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.sh
7178 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
7179
7180 // AVX512 FP16 Arithmetic
7181 case Intrinsic::x86_avx512fp16_mask_add_sh_round:
7182 case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
7183 case Intrinsic::x86_avx512fp16_mask_mul_sh_round:
7184 case Intrinsic::x86_avx512fp16_mask_div_sh_round:
7185 case Intrinsic::x86_avx512fp16_mask_max_sh_round:
7186 case Intrinsic::x86_avx512fp16_mask_min_sh_round: {
7187 visitGenericScalarHalfwordInst(I);
7188 break;
7189 }
7190
7191 // AVX512 Floating-Point Classification
7192 // - <8 x i1> @llvm.x86.avx512.fpclass.pd.512(<8 x double>, i32)
7193 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512(<16 x float>, i32)
7194 case Intrinsic::x86_avx512_fpclass_pd_512:
7195 case Intrinsic::x86_avx512_fpclass_ps_512:
7196 handleAVX512FPClass(I);
7197 break;
7198
7199 // AVX Galois Field New Instructions
7200 case Intrinsic::x86_vgf2p8affineqb_128:
7201 case Intrinsic::x86_vgf2p8affineqb_256:
7202 case Intrinsic::x86_vgf2p8affineqb_512:
7203 handleAVXGF2P8Affine(I);
7204 break;
7205
7206 default:
7207 return false;
7208 }
7209
7210 return true;
7211 }
7212
7213 bool maybeHandleArmSIMDIntrinsic(IntrinsicInst &I) {
7214 switch (I.getIntrinsicID()) {
7215 // Two operands e.g.,
7216 // - <8 x i8> @llvm.aarch64.neon.rshrn.v8i8 (<8 x i16>, i32)
7217 // - <4 x i16> @llvm.aarch64.neon.uqrshl.v4i16(<4 x i16>, <4 x i16>)
7218 case Intrinsic::aarch64_neon_rshrn:
7219 case Intrinsic::aarch64_neon_sqrshl:
7220 case Intrinsic::aarch64_neon_sqrshrn:
7221 case Intrinsic::aarch64_neon_sqrshrun:
7222 case Intrinsic::aarch64_neon_sqshl:
7223 case Intrinsic::aarch64_neon_sqshlu:
7224 case Intrinsic::aarch64_neon_sqshrn:
7225 case Intrinsic::aarch64_neon_sqshrun:
7226 case Intrinsic::aarch64_neon_srshl:
7227 case Intrinsic::aarch64_neon_sshl:
7228 case Intrinsic::aarch64_neon_uqrshl:
7229 case Intrinsic::aarch64_neon_uqrshrn:
7230 case Intrinsic::aarch64_neon_uqshl:
7231 case Intrinsic::aarch64_neon_uqshrn:
7232 case Intrinsic::aarch64_neon_urshl:
7233 case Intrinsic::aarch64_neon_ushl:
7234 handleVectorShiftIntrinsic(I, /* Variable */ false);
7235 break;
7236
7237 // Vector Shift Left/Right and Insert
7238 //
7239 // Three operands e.g.,
7240 // - <4 x i16> @llvm.aarch64.neon.vsli.v4i16
7241 // (<4 x i16> %a, <4 x i16> %b, i32 %n)
7242 // - <16 x i8> @llvm.aarch64.neon.vsri.v16i8
7243 // (<16 x i8> %a, <16 x i8> %b, i32 %n)
7244 //
7245 // %b is shifted by %n bits, and the "missing" bits are filled in with %a
7246 // (instead of zero-extending/sign-extending).
7247 case Intrinsic::aarch64_neon_vsli:
7248 case Intrinsic::aarch64_neon_vsri:
7249 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7250 /*trailingVerbatimArgs=*/1,
7251 /*forceIntegerIntrinsic=*/false);
7252 break;
7253
7254 // TODO: handling max/min similarly to AND/OR may be more precise
7255 // Floating-Point Maximum/Minimum Pairwise
7256 case Intrinsic::aarch64_neon_fmaxp:
7257 case Intrinsic::aarch64_neon_fminp:
7258 // Floating-Point Maximum/Minimum Number Pairwise
7259 case Intrinsic::aarch64_neon_fmaxnmp:
7260 case Intrinsic::aarch64_neon_fminnmp:
7261 // Signed/Unsigned Maximum/Minimum Pairwise
7262 case Intrinsic::aarch64_neon_smaxp:
7263 case Intrinsic::aarch64_neon_sminp:
7264 case Intrinsic::aarch64_neon_umaxp:
7265 case Intrinsic::aarch64_neon_uminp:
7266 // Add Pairwise
7267 case Intrinsic::aarch64_neon_addp:
7268 // Floating-point Add Pairwise
7269 case Intrinsic::aarch64_neon_faddp:
7270 // Add Long Pairwise
7271 case Intrinsic::aarch64_neon_saddlp:
7272 case Intrinsic::aarch64_neon_uaddlp: {
7273 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
7274 break;
7275 }
7276
7277 // Floating-point Convert to integer, rounding to nearest with ties to Away
7278 case Intrinsic::aarch64_neon_fcvtas:
7279 case Intrinsic::aarch64_neon_fcvtau:
7280 // Floating-point convert to integer, rounding toward minus infinity
7281 case Intrinsic::aarch64_neon_fcvtms:
7282 case Intrinsic::aarch64_neon_fcvtmu:
7283 // Floating-point convert to integer, rounding to nearest with ties to even
7284 case Intrinsic::aarch64_neon_fcvtns:
7285 case Intrinsic::aarch64_neon_fcvtnu:
7286 // Floating-point convert to integer, rounding toward plus infinity
7287 case Intrinsic::aarch64_neon_fcvtps:
7288 case Intrinsic::aarch64_neon_fcvtpu:
7289 // Floating-point Convert to integer, rounding toward Zero
7290 case Intrinsic::aarch64_neon_fcvtzs:
7291 case Intrinsic::aarch64_neon_fcvtzu:
7292 // Floating-point convert to lower precision narrow, rounding to odd
7293 case Intrinsic::aarch64_neon_fcvtxn:
7294 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
7295 break;
7296
7297 // Vector Conversions Between Fixed-Point and Floating-Point
7298 case Intrinsic::aarch64_neon_vcvtfxs2fp:
7299 case Intrinsic::aarch64_neon_vcvtfp2fxs:
7300 case Intrinsic::aarch64_neon_vcvtfxu2fp:
7301 case Intrinsic::aarch64_neon_vcvtfp2fxu:
7302 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/true);
7303 break;
7304
7305 // TODO: bfloat conversions
7306 // - bfloat @llvm.aarch64.neon.bfcvt(float)
7307 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float>)
7308 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat>, <4 x float>)
7309
7310 // Add reduction to scalar
7311 case Intrinsic::aarch64_neon_faddv:
7312 case Intrinsic::aarch64_neon_saddv:
7313 case Intrinsic::aarch64_neon_uaddv:
7314 // Signed/Unsigned min/max (Vector)
7315 // TODO: handling similarly to AND/OR may be more precise.
7316 case Intrinsic::aarch64_neon_smaxv:
7317 case Intrinsic::aarch64_neon_sminv:
7318 case Intrinsic::aarch64_neon_umaxv:
7319 case Intrinsic::aarch64_neon_uminv:
7320 // Floating-point min/max (vector)
7321 // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
7322 // but our shadow propagation is the same.
7323 case Intrinsic::aarch64_neon_fmaxv:
7324 case Intrinsic::aarch64_neon_fminv:
7325 case Intrinsic::aarch64_neon_fmaxnmv:
7326 case Intrinsic::aarch64_neon_fminnmv:
7327 // Sum long across vector
7328 case Intrinsic::aarch64_neon_saddlv:
7329 case Intrinsic::aarch64_neon_uaddlv:
7330 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/true);
7331 break;
7332
7333 case Intrinsic::aarch64_neon_ld1x2:
7334 case Intrinsic::aarch64_neon_ld1x3:
7335 case Intrinsic::aarch64_neon_ld1x4:
7336 case Intrinsic::aarch64_neon_ld2:
7337 case Intrinsic::aarch64_neon_ld3:
7338 case Intrinsic::aarch64_neon_ld4:
7339 case Intrinsic::aarch64_neon_ld2r:
7340 case Intrinsic::aarch64_neon_ld3r:
7341 case Intrinsic::aarch64_neon_ld4r: {
7342 handleNEONVectorLoad(I, /*WithLane=*/false);
7343 break;
7344 }
7345
7346 case Intrinsic::aarch64_neon_ld2lane:
7347 case Intrinsic::aarch64_neon_ld3lane:
7348 case Intrinsic::aarch64_neon_ld4lane: {
7349 handleNEONVectorLoad(I, /*WithLane=*/true);
7350 break;
7351 }
7352
7353 // Saturating extract narrow
7354 case Intrinsic::aarch64_neon_sqxtn:
7355 case Intrinsic::aarch64_neon_sqxtun:
7356 case Intrinsic::aarch64_neon_uqxtn:
7357 // These only have one argument, but we (ab)use handleShadowOr because it
7358 // does work on single argument intrinsics and will typecast the shadow
7359 // (and update the origin).
7360 handleShadowOr(I);
7361 break;
7362
7363 case Intrinsic::aarch64_neon_st1x2:
7364 case Intrinsic::aarch64_neon_st1x3:
7365 case Intrinsic::aarch64_neon_st1x4:
7366 case Intrinsic::aarch64_neon_st2:
7367 case Intrinsic::aarch64_neon_st3:
7368 case Intrinsic::aarch64_neon_st4: {
7369 handleNEONVectorStoreIntrinsic(I, useLane: false);
7370 break;
7371 }
7372
7373 case Intrinsic::aarch64_neon_st2lane:
7374 case Intrinsic::aarch64_neon_st3lane:
7375 case Intrinsic::aarch64_neon_st4lane: {
7376 handleNEONVectorStoreIntrinsic(I, useLane: true);
7377 break;
7378 }
7379
7380 // Arm NEON vector table intrinsics have the source/table register(s) as
7381 // arguments, followed by the index register. They return the output.
7382 //
7383 // 'TBL writes a zero if an index is out-of-range, while TBX leaves the
7384 // original value unchanged in the destination register.'
7385 // Conveniently, zero denotes a clean shadow, which means out-of-range
7386 // indices for TBL will initialize the user data with zero and also clean
7387 // the shadow. (For TBX, neither the user data nor the shadow will be
7388 // updated, which is also correct.)
7389 case Intrinsic::aarch64_neon_tbl1:
7390 case Intrinsic::aarch64_neon_tbl2:
7391 case Intrinsic::aarch64_neon_tbl3:
7392 case Intrinsic::aarch64_neon_tbl4:
7393 case Intrinsic::aarch64_neon_tbx1:
7394 case Intrinsic::aarch64_neon_tbx2:
7395 case Intrinsic::aarch64_neon_tbx3:
7396 case Intrinsic::aarch64_neon_tbx4: {
7397 // The last trailing argument (index register) should be handled verbatim
7398 handleIntrinsicByApplyingToShadow(
7399 I, /*shadowIntrinsicID=*/I.getIntrinsicID(),
7400 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
7401 break;
7402 }
7403
7404 case Intrinsic::aarch64_neon_fmulx:
7405 case Intrinsic::aarch64_neon_pmul:
7406 case Intrinsic::aarch64_neon_pmull:
7407 case Intrinsic::aarch64_neon_smull:
7408 case Intrinsic::aarch64_neon_pmull64:
7409 case Intrinsic::aarch64_neon_umull: {
7410 handleNEONVectorMultiplyIntrinsic(I);
7411 break;
7412 }
7413
7414 case Intrinsic::aarch64_neon_smmla:
7415 case Intrinsic::aarch64_neon_ummla:
7416 case Intrinsic::aarch64_neon_usmmla:
7417 case Intrinsic::aarch64_neon_bfmmla:
7418 handleNEONMatrixMultiply(I);
7419 break;
7420
7421 // <2 x i32> @llvm.aarch64.neon.{u,s,us}dot.v2i32.v8i8
7422 // (<2 x i32> %acc, <8 x i8> %a, <8 x i8> %b)
7423 // <4 x i32> @llvm.aarch64.neon.{u,s,us}dot.v4i32.v16i8
7424 // (<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b)
7425 case Intrinsic::aarch64_neon_sdot:
7426 case Intrinsic::aarch64_neon_udot:
7427 case Intrinsic::aarch64_neon_usdot:
7428 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7429 /*ZeroPurifies=*/true,
7430 /*EltSizeInBits=*/0,
7431 /*Lanes=*/kBothLanes);
7432 break;
7433
7434 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
7435 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
7436 // <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16
7437 // (<4 x float> %acc, <8 x bfloat> %a, <8 x bfloat> %b)
7438 case Intrinsic::aarch64_neon_bfdot:
7439 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7440 /*ZeroPurifies=*/false,
7441 /*EltSizeInBits=*/0,
7442 /*Lanes=*/kBothLanes);
7443 break;
7444
7445 // Floating-Point Absolute Compare Greater Than/Equal
7446 case Intrinsic::aarch64_neon_facge:
7447 case Intrinsic::aarch64_neon_facgt:
7448 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/false);
7449 break;
7450
7451 default:
7452 return false;
7453 }
7454
7455 return true;
7456 }
7457
7458 void visitIntrinsicInst(IntrinsicInst &I) {
7459 if (maybeHandleCrossPlatformIntrinsic(I))
7460 return;
7461
7462 if (maybeHandleX86SIMDIntrinsic(I))
7463 return;
7464
7465 if (maybeHandleArmSIMDIntrinsic(I))
7466 return;
7467
7468 if (maybeHandleUnknownIntrinsic(I))
7469 return;
7470
7471 visitInstruction(I);
7472 }
7473
7474 void visitLibAtomicLoad(CallBase &CB) {
7475 // Since we use getNextNode here, we can't have CB terminate the BB.
7476 assert(isa<CallInst>(CB));
7477
7478 IRBuilder<> IRB(&CB);
7479 Value *Size = CB.getArgOperand(i: 0);
7480 Value *SrcPtr = CB.getArgOperand(i: 1);
7481 Value *DstPtr = CB.getArgOperand(i: 2);
7482 Value *Ordering = CB.getArgOperand(i: 3);
7483 // Convert the call to have at least Acquire ordering to make sure
7484 // the shadow operations aren't reordered before it.
7485 Value *NewOrdering =
7486 IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
7487 CB.setArgOperand(i: 3, v: NewOrdering);
7488
7489 NextNodeIRBuilder NextIRB(&CB);
7490 Value *SrcShadowPtr, *SrcOriginPtr;
7491 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
7492 getShadowOriginPtr(Addr: SrcPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7493 /*isStore*/ false);
7494 Value *DstShadowPtr =
7495 getShadowOriginPtr(Addr: DstPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7496 /*isStore*/ true)
7497 .first;
7498
7499 NextIRB.CreateMemCpy(Dst: DstShadowPtr, DstAlign: Align(1), Src: SrcShadowPtr, SrcAlign: Align(1), Size);
7500 if (MS.TrackOrigins) {
7501 Value *SrcOrigin = NextIRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr,
7502 Align: kMinOriginAlignment);
7503 Value *NewOrigin = updateOrigin(V: SrcOrigin, IRB&: NextIRB);
7504 NextIRB.CreateCall(Callee: MS.MsanSetOriginFn, Args: {DstPtr, Size, NewOrigin});
7505 }
7506 }
7507
7508 void visitLibAtomicStore(CallBase &CB) {
7509 IRBuilder<> IRB(&CB);
7510 Value *Size = CB.getArgOperand(i: 0);
7511 Value *DstPtr = CB.getArgOperand(i: 2);
7512 Value *Ordering = CB.getArgOperand(i: 3);
7513 // Convert the call to have at least Release ordering to make sure
7514 // the shadow operations aren't reordered after it.
7515 Value *NewOrdering =
7516 IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
7517 CB.setArgOperand(i: 3, v: NewOrdering);
7518
7519 Value *DstShadowPtr =
7520 getShadowOriginPtr(Addr: DstPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1),
7521 /*isStore*/ true)
7522 .first;
7523
7524 // Atomic store always paints clean shadow/origin. See file header.
7525 IRB.CreateMemSet(Ptr: DstShadowPtr, Val: getCleanShadow(OrigTy: IRB.getInt8Ty()), Size,
7526 Align: Align(1));
7527 }
7528
7529 void visitCallBase(CallBase &CB) {
7530 assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
7531 if (CB.isInlineAsm()) {
7532 // For inline asm (either a call to asm function, or callbr instruction),
7533 // do the usual thing: check argument shadow and mark all outputs as
7534 // clean. Note that any side effects of the inline asm that are not
7535 // immediately visible in its constraints are not handled.
7536 if (ClHandleAsmConservative)
7537 visitAsmInstruction(I&: CB);
7538 else
7539 visitInstruction(I&: CB);
7540 return;
7541 }
7542 LibFunc LF = TLI->getLibFunc(CB);
7543 if (LF != NotLibFunc) {
7544 // libatomic.a functions need to have special handling because there isn't
7545 // a good way to intercept them or compile the library with
7546 // instrumentation.
7547 switch (LF) {
7548 case LibFunc_atomic_load:
7549 if (!isa<CallInst>(Val: CB)) {
7550 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
7551 "Ignoring!\n";
7552 break;
7553 }
7554 visitLibAtomicLoad(CB);
7555 return;
7556 case LibFunc_atomic_store:
7557 visitLibAtomicStore(CB);
7558 return;
7559 default:
7560 break;
7561 }
7562 }
7563
7564 if (auto *Call = dyn_cast<CallInst>(Val: &CB)) {
7565 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
7566
7567 // We are going to insert code that relies on the fact that the callee
7568 // will become a non-readonly function after it is instrumented by us. To
7569 // prevent this code from being optimized out, mark that function
7570 // non-readonly in advance.
7571 // TODO: We can likely do better than dropping memory() completely here.
7572 AttributeMask B;
7573 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
7574
7575 Call->removeFnAttrs(AttrsToRemove: B);
7576 if (Function *Func = Call->getCalledFunction()) {
7577 Func->removeFnAttrs(Attrs: B);
7578 }
7579
7580 maybeMarkSanitizerLibraryCallNoBuiltin(CI: Call, TLI);
7581 }
7582 IRBuilder<> IRB(&CB);
7583 bool MayCheckCall = MS.EagerChecks;
7584 if (Function *Func = CB.getCalledFunction()) {
7585 // __sanitizer_unaligned_{load,store} functions may be called by users
7586 // and always expects shadows in the TLS. So don't check them.
7587 MayCheckCall &= !Func->getName().starts_with(Prefix: "__sanitizer_unaligned_");
7588 }
7589
7590 unsigned ArgOffset = 0;
7591 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
7592 for (const auto &[i, A] : llvm::enumerate(First: CB.args())) {
7593 if (!A->getType()->isSized()) {
7594 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
7595 continue;
7596 }
7597
7598 if (A->getType()->isScalableTy()) {
7599 LLVM_DEBUG(dbgs() << "Arg " << i << " is vscale: " << CB << "\n");
7600 // Handle as noundef, but don't reserve tls slots.
7601 insertCheckShadowOf(Val: A, OrigIns: &CB);
7602 continue;
7603 }
7604
7605 unsigned Size = 0;
7606 const DataLayout &DL = F.getDataLayout();
7607
7608 bool ByVal = CB.paramHasAttr(ArgNo: i, Kind: Attribute::ByVal);
7609 bool NoUndef = CB.paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef);
7610 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
7611
7612 if (EagerCheck) {
7613 insertCheckShadowOf(Val: A, OrigIns: &CB);
7614 Size = DL.getTypeAllocSize(Ty: A->getType());
7615 } else {
7616 [[maybe_unused]] Value *Store = nullptr;
7617 // Compute the Shadow for arg even if it is ByVal, because
7618 // in that case getShadow() will copy the actual arg shadow to
7619 // __msan_param_tls.
7620 Value *ArgShadow = getShadow(V: A);
7621 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
7622 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
7623 << " Shadow: " << *ArgShadow << "\n");
7624 if (ByVal) {
7625 // ByVal requires some special handling as it's too big for a single
7626 // load
7627 assert(A->getType()->isPointerTy() &&
7628 "ByVal argument is not a pointer!");
7629 Size = DL.getTypeAllocSize(Ty: CB.getParamByValType(ArgNo: i));
7630 if (ArgOffset + Size > kParamTLSSize)
7631 break;
7632 const MaybeAlign ParamAlignment(CB.getParamAlign(ArgNo: i));
7633 MaybeAlign Alignment = std::nullopt;
7634 if (ParamAlignment)
7635 Alignment = std::min(a: *ParamAlignment, b: kShadowTLSAlignment);
7636 Value *AShadowPtr, *AOriginPtr;
7637 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
7638 getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
7639 /*isStore*/ false);
7640 if (!PropagateShadow) {
7641 Store = IRB.CreateMemSet(Ptr: ArgShadowBase,
7642 Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
7643 Size, Align: Alignment);
7644 } else {
7645 Store = IRB.CreateMemCpy(Dst: ArgShadowBase, DstAlign: Alignment, Src: AShadowPtr,
7646 SrcAlign: Alignment, Size);
7647 if (MS.TrackOrigins) {
7648 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
7649 // FIXME: OriginSize should be:
7650 // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
7651 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
7652 IRB.CreateMemCpy(
7653 Dst: ArgOriginBase,
7654 /* by origin_tls[ArgOffset] */ DstAlign: kMinOriginAlignment,
7655 Src: AOriginPtr,
7656 /* by getShadowOriginPtr */ SrcAlign: kMinOriginAlignment, Size: OriginSize);
7657 }
7658 }
7659 } else {
7660 // Any other parameters mean we need bit-grained tracking of uninit
7661 // data
7662 Size = DL.getTypeAllocSize(Ty: A->getType());
7663 if (ArgOffset + Size > kParamTLSSize)
7664 break;
7665 Store = IRB.CreateAlignedStore(Val: ArgShadow, Ptr: ArgShadowBase,
7666 Align: kShadowTLSAlignment);
7667 Constant *Cst = dyn_cast<Constant>(Val: ArgShadow);
7668 if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
7669 IRB.CreateStore(Val: getOrigin(V: A),
7670 Ptr: getOriginPtrForArgument(IRB, ArgOffset));
7671 }
7672 }
7673 assert(Store != nullptr);
7674 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
7675 }
7676 assert(Size != 0);
7677 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
7678 }
7679 LLVM_DEBUG(dbgs() << " done with call args\n");
7680
7681 FunctionType *FT = CB.getFunctionType();
7682 if (FT->isVarArg()) {
7683 VAHelper->visitCallBase(CB, IRB);
7684 }
7685
7686 // Now, get the shadow for the RetVal.
7687 if (!CB.getType()->isSized())
7688 return;
7689 // Don't emit the epilogue for musttail call returns.
7690 if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
7691 return;
7692
7693 if (MayCheckCall && CB.hasRetAttr(Kind: Attribute::NoUndef)) {
7694 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7695 setOrigin(V: &CB, Origin: getCleanOrigin());
7696 return;
7697 }
7698
7699 IRBuilder<> IRBBefore(&CB);
7700 // Until we have full dynamic coverage, make sure the retval shadow is 0.
7701 Value *Base = getShadowPtrForRetval(IRB&: IRBBefore);
7702 IRBBefore.CreateAlignedStore(Val: getCleanShadow(V: &CB), Ptr: Base,
7703 Align: kShadowTLSAlignment);
7704 BasicBlock::iterator NextInsn;
7705 if (isa<CallInst>(Val: CB)) {
7706 NextInsn = ++CB.getIterator();
7707 assert(NextInsn != CB.getParent()->end());
7708 } else {
7709 BasicBlock *NormalDest = cast<InvokeInst>(Val&: CB).getNormalDest();
7710 if (!NormalDest->getSinglePredecessor()) {
7711 // FIXME: this case is tricky, so we are just conservative here.
7712 // Perhaps we need to split the edge between this BB and NormalDest,
7713 // but a naive attempt to use SplitEdge leads to a crash.
7714 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7715 setOrigin(V: &CB, Origin: getCleanOrigin());
7716 return;
7717 }
7718 // FIXME: NextInsn is likely in a basic block that has not been visited
7719 // yet. Anything inserted there will be instrumented by MSan later!
7720 NextInsn = NormalDest->getFirstInsertionPt();
7721 assert(NextInsn != NormalDest->end() &&
7722 "Could not find insertion point for retval shadow load");
7723 }
7724 IRBuilder<> IRBAfter(&*NextInsn);
7725 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
7726 Ty: getShadowTy(V: &CB), Ptr: getShadowPtrForRetval(IRB&: IRBAfter), Align: kShadowTLSAlignment,
7727 Name: "_msret");
7728 setShadow(V: &CB, SV: RetvalShadow);
7729 if (MS.TrackOrigins)
7730 setOrigin(V: &CB, Origin: IRBAfter.CreateLoad(Ty: MS.OriginTy, Ptr: getOriginPtrForRetval()));
7731 }
7732
7733 bool isAMustTailRetVal(Value *RetVal) {
7734 if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
7735 RetVal = I->getOperand(i_nocapture: 0);
7736 }
7737 if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
7738 return I->isMustTailCall();
7739 }
7740 return false;
7741 }
7742
7743 void visitReturnInst(ReturnInst &I) {
7744 IRBuilder<> IRB(&I);
7745 Value *RetVal = I.getReturnValue();
7746 if (!RetVal)
7747 return;
7748 // Don't emit the epilogue for musttail call returns.
7749 if (isAMustTailRetVal(RetVal))
7750 return;
7751 Value *ShadowPtr = getShadowPtrForRetval(IRB);
7752 bool HasNoUndef = F.hasRetAttribute(Kind: Attribute::NoUndef);
7753 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
7754 // FIXME: Consider using SpecialCaseList to specify a list of functions that
7755 // must always return fully initialized values. For now, we hardcode "main".
7756 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
7757
7758 Value *Shadow = getShadow(V: RetVal);
7759 bool StoreOrigin = true;
7760 if (EagerCheck) {
7761 insertCheckShadowOf(Val: RetVal, OrigIns: &I);
7762 Shadow = getCleanShadow(V: RetVal);
7763 StoreOrigin = false;
7764 }
7765
7766 // The caller may still expect information passed over TLS if we pass our
7767 // check
7768 if (StoreShadow) {
7769 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: kShadowTLSAlignment);
7770 if (MS.TrackOrigins && StoreOrigin)
7771 IRB.CreateStore(Val: getOrigin(V: RetVal), Ptr: getOriginPtrForRetval());
7772 }
7773 }
7774
7775 void visitPHINode(PHINode &I) {
7776 IRBuilder<> IRB(&I);
7777 if (!PropagateShadow) {
7778 setShadow(V: &I, SV: getCleanShadow(V: &I));
7779 setOrigin(V: &I, Origin: getCleanOrigin());
7780 return;
7781 }
7782
7783 ShadowPHINodes.push_back(Elt: &I);
7784 setShadow(V: &I, SV: IRB.CreatePHI(Ty: getShadowTy(V: &I), NumReservedValues: I.getNumIncomingValues(),
7785 Name: "_msphi_s"));
7786 if (MS.TrackOrigins)
7787 setOrigin(
7788 V: &I, Origin: IRB.CreatePHI(Ty: MS.OriginTy, NumReservedValues: I.getNumIncomingValues(), Name: "_msphi_o"));
7789 }
7790
7791 Value *getLocalVarIdptr(AllocaInst &I) {
7792 ConstantInt *IntConst =
7793 ConstantInt::get(Ty: Type::getInt32Ty(C&: (*F.getParent()).getContext()), V: 0);
7794 return new GlobalVariable(*F.getParent(), IntConst->getType(),
7795 /*isConstant=*/false, GlobalValue::PrivateLinkage,
7796 IntConst);
7797 }
7798
7799 Value *getLocalVarDescription(AllocaInst &I) {
7800 return createPrivateConstGlobalForString(M&: *F.getParent(), Str: I.getName());
7801 }
7802
7803 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7804 if (PoisonStack && ClPoisonStackWithCall) {
7805 IRB.CreateCall(Callee: MS.MsanPoisonStackFn, Args: {&I, Len});
7806 } else {
7807 Value *ShadowBase, *OriginBase;
7808 std::tie(args&: ShadowBase, args&: OriginBase) = getShadowOriginPtr(
7809 Addr: &I, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1), /*isStore*/ true);
7810
7811 Value *PoisonValue = IRB.getInt8(C: PoisonStack ? ClPoisonStackPattern : 0);
7812 IRB.CreateMemSet(Ptr: ShadowBase, Val: PoisonValue, Size: Len, Align: I.getAlign());
7813 }
7814
7815 if (PoisonStack && MS.TrackOrigins) {
7816 Value *Idptr = getLocalVarIdptr(I);
7817 if (ClPrintStackNames) {
7818 Value *Descr = getLocalVarDescription(I);
7819 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginWithDescriptionFn,
7820 Args: {&I, Len, Idptr, Descr});
7821 } else {
7822 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginNoDescriptionFn, Args: {&I, Len, Idptr});
7823 }
7824 }
7825 }
7826
7827 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7828 Value *Descr = getLocalVarDescription(I);
7829 if (PoisonStack) {
7830 IRB.CreateCall(Callee: MS.MsanPoisonAllocaFn, Args: {&I, Len, Descr});
7831 } else {
7832 IRB.CreateCall(Callee: MS.MsanUnpoisonAllocaFn, Args: {&I, Len});
7833 }
7834 }
7835
7836 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
7837 if (!InsPoint)
7838 InsPoint = &I;
7839 NextNodeIRBuilder IRB(InsPoint);
7840 Value *Len = IRB.CreateAllocationSize(DestTy: MS.IntptrTy, AI: &I);
7841
7842 if (MS.CompileKernel)
7843 poisonAllocaKmsan(I, IRB, Len);
7844 else
7845 poisonAllocaUserspace(I, IRB, Len);
7846 }
7847
7848 void visitAllocaInst(AllocaInst &I) {
7849 setShadow(V: &I, SV: getCleanShadow(V: &I));
7850 setOrigin(V: &I, Origin: getCleanOrigin());
7851 // We'll get to this alloca later unless it's poisoned at the corresponding
7852 // llvm.lifetime.start.
7853 AllocaSet.insert(X: &I);
7854 }
7855
7856 void visitSelectInst(SelectInst &I) {
7857 // a = select b, c, d
7858 Value *B = I.getCondition();
7859 Value *C = I.getTrueValue();
7860 Value *D = I.getFalseValue();
7861
7862 handleSelectLikeInst(I, B, C, D);
7863 }
7864
7865 void handleSelectLikeInst(Instruction &I, Value *B, Value *C, Value *D) {
7866 IRBuilder<> IRB(&I);
7867
7868 Value *Sb = getShadow(V: B);
7869 Value *Sc = getShadow(V: C);
7870 Value *Sd = getShadow(V: D);
7871
7872 Value *Ob = MS.TrackOrigins ? getOrigin(V: B) : nullptr;
7873 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
7874 Value *Od = MS.TrackOrigins ? getOrigin(V: D) : nullptr;
7875
7876 // Result shadow if condition shadow is 0.
7877 Value *Sa0 = IRB.CreateSelect(C: B, True: Sc, False: Sd);
7878 Value *Sa1;
7879 if (I.getType()->isAggregateType()) {
7880 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
7881 // an extra "select". This results in much more compact IR.
7882 // Sa = select Sb, poisoned, (select b, Sc, Sd)
7883 Sa1 = getPoisonedShadow(ShadowTy: getShadowTy(OrigTy: I.getType()));
7884 } else if (isScalableNonVectorType(Ty: I.getType())) {
7885 // This is intended to handle target("aarch64.svcount"), which can't be
7886 // handled in the else branch because of incompatibility with CreateXor
7887 // ("The supported LLVM operations on this type are limited to load,
7888 // store, phi, select and alloca instructions").
7889
7890 // TODO: this currently underapproximates. Use Arm SVE EOR in the else
7891 // branch as needed instead.
7892 Sa1 = getCleanShadow(OrigTy: getShadowTy(OrigTy: I.getType()));
7893 } else {
7894 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
7895 // If Sb (condition is poisoned), look for bits in c and d that are equal
7896 // and both unpoisoned.
7897 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
7898
7899 // Cast arguments to shadow-compatible type.
7900 C = CreateAppToShadowCast(IRB, V: C);
7901 D = CreateAppToShadowCast(IRB, V: D);
7902
7903 // Result shadow if condition shadow is 1.
7904 Sa1 = IRB.CreateOr(Ops: {IRB.CreateXor(LHS: C, RHS: D), Sc, Sd});
7905 }
7906 Value *Sa = IRB.CreateSelect(C: Sb, True: Sa1, False: Sa0, Name: "_msprop_select");
7907 setShadow(V: &I, SV: Sa);
7908 if (MS.TrackOrigins) {
7909 // Origins are always i32, so any vector conditions must be flattened.
7910 // FIXME: consider tracking vector origins for app vectors?
7911 if (B->getType()->isVectorTy()) {
7912 B = convertToBool(V: B, IRB);
7913 Sb = convertToBool(V: Sb, IRB);
7914 }
7915 // a = select b, c, d
7916 // Oa = Sb ? Ob : (b ? Oc : Od)
7917 setOrigin(V: &I, Origin: IRB.CreateSelect(C: Sb, True: Ob, False: IRB.CreateSelect(C: B, True: Oc, False: Od)));
7918 }
7919 }
7920
7921 void visitLandingPadInst(LandingPadInst &I) {
7922 // Do nothing.
7923 // See https://github.com/google/sanitizers/issues/504
7924 setShadow(V: &I, SV: getCleanShadow(V: &I));
7925 setOrigin(V: &I, Origin: getCleanOrigin());
7926 }
7927
7928 void visitCatchSwitchInst(CatchSwitchInst &I) {
7929 setShadow(V: &I, SV: getCleanShadow(V: &I));
7930 setOrigin(V: &I, Origin: getCleanOrigin());
7931 }
7932
7933 void visitFuncletPadInst(FuncletPadInst &I) {
7934 setShadow(V: &I, SV: getCleanShadow(V: &I));
7935 setOrigin(V: &I, Origin: getCleanOrigin());
7936 }
7937
7938 void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
7939
7940 void visitExtractValueInst(ExtractValueInst &I) {
7941 IRBuilder<> IRB(&I);
7942 Value *Agg = I.getAggregateOperand();
7943 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
7944 Value *AggShadow = getShadow(V: Agg);
7945 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
7946 Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
7947 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
7948 setShadow(V: &I, SV: ResShadow);
7949 setOriginForNaryOp(I);
7950 }
7951
7952 void visitInsertValueInst(InsertValueInst &I) {
7953 IRBuilder<> IRB(&I);
7954 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
7955 Value *AggShadow = getShadow(V: I.getAggregateOperand());
7956 Value *InsShadow = getShadow(V: I.getInsertedValueOperand());
7957 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
7958 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
7959 Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
7960 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
7961 setShadow(V: &I, SV: Res);
7962 setOriginForNaryOp(I);
7963 }
7964
7965 void dumpInst(Instruction &I, const Twine &Prefix) {
7966 // Instruction name only
7967 // For intrinsics, the full/overloaded name is used
7968 //
7969 // e.g., "call llvm.aarch64.neon.uqsub.v16i8"
7970 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
7971 errs() << "ZZZ:" << Prefix << " call "
7972 << CI->getCalledFunction()->getName() << "\n";
7973 } else {
7974 errs() << "ZZZ:" << Prefix << " " << I.getOpcodeName() << "\n";
7975 }
7976
7977 // Instruction prototype (including return type and parameter types)
7978 // For intrinsics, we use the base/non-overloaded name
7979 //
7980 // e.g., "call <16 x i8> @llvm.aarch64.neon.uqsub(<16 x i8>, <16 x i8>)"
7981 unsigned NumOperands = I.getNumOperands();
7982 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
7983 errs() << "YYY:" << Prefix << " call " << *I.getType() << " @";
7984
7985 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI))
7986 errs() << Intrinsic::getBaseName(id: II->getIntrinsicID());
7987 else
7988 errs() << CI->getCalledFunction()->getName();
7989
7990 errs() << "(";
7991
7992 // The last operand of a CallInst is the function itself.
7993 NumOperands--;
7994 } else
7995 errs() << "YYY:" << Prefix << " " << *I.getType() << " "
7996 << I.getOpcodeName() << "(";
7997
7998 for (size_t i = 0; i < NumOperands; i++) {
7999 if (i > 0)
8000 errs() << ", ";
8001
8002 errs() << *(I.getOperand(i)->getType());
8003 }
8004
8005 errs() << ")\n";
8006
8007 // Full instruction, including types and operand values
8008 // For intrinsics, the full/overloaded name is used
8009 //
8010 // e.g., "%vqsubq_v.i15 = call noundef <16 x i8>
8011 // @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %vext21.i,
8012 // <16 x i8> splat (i8 1)), !dbg !66"
8013 errs() << "QQQ:" << Prefix << " " << I << "\n";
8014 }
8015
8016 void visitResumeInst(ResumeInst &I) {
8017 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
8018 // Nothing to do here.
8019 }
8020
8021 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
8022 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
8023 // Nothing to do here.
8024 }
8025
8026 void visitCatchReturnInst(CatchReturnInst &CRI) {
8027 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
8028 // Nothing to do here.
8029 }
8030
8031 void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
8032 IRBuilder<> &IRB, const DataLayout &DL,
8033 bool isOutput) {
8034 // For each assembly argument, we check its value for being initialized.
8035 // If the argument is a pointer, we assume it points to a single element
8036 // of the corresponding type (or to a 8-byte word, if the type is unsized).
8037 // Each such pointer is instrumented with a call to the runtime library.
8038 Type *OpType = Operand->getType();
8039 // Check the operand value itself.
8040 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8041 if (!OpType->isPointerTy() || !isOutput) {
8042 assert(!isOutput);
8043 return;
8044 }
8045 if (!ElemTy->isSized())
8046 return;
8047 auto Size = DL.getTypeStoreSize(Ty: ElemTy);
8048 Value *SizeVal = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size);
8049 if (MS.CompileKernel) {
8050 IRB.CreateCall(Callee: MS.MsanInstrumentAsmStoreFn, Args: {Operand, SizeVal});
8051 } else {
8052 // ElemTy, derived from elementtype(), does not encode the alignment of
8053 // the pointer. Conservatively assume that the shadow memory is unaligned.
8054 // When Size is large, avoid StoreInst as it would expand to many
8055 // instructions.
8056 auto [ShadowPtr, _] =
8057 getShadowOriginPtrUserspace(Addr: Operand, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1));
8058 if (Size <= 32)
8059 IRB.CreateAlignedStore(Val: getCleanShadow(OrigTy: ElemTy), Ptr: ShadowPtr, Align: Align(1));
8060 else
8061 IRB.CreateMemSet(Ptr: ShadowPtr, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8062 Size: SizeVal, Align: Align(1));
8063 }
8064 }
8065
8066 /// Get the number of output arguments returned by pointers.
8067 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
8068 int NumRetOutputs = 0;
8069 int NumOutputs = 0;
8070 Type *RetTy = cast<Value>(Val: CB)->getType();
8071 if (!RetTy->isVoidTy()) {
8072 // Register outputs are returned via the CallInst return value.
8073 auto *ST = dyn_cast<StructType>(Val: RetTy);
8074 if (ST)
8075 NumRetOutputs = ST->getNumElements();
8076 else
8077 NumRetOutputs = 1;
8078 }
8079 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
8080 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
8081 switch (Info.Type) {
8082 case InlineAsm::isOutput:
8083 NumOutputs++;
8084 break;
8085 default:
8086 break;
8087 }
8088 }
8089 return NumOutputs - NumRetOutputs;
8090 }
8091
8092 void visitAsmInstruction(Instruction &I) {
8093 // Conservative inline assembly handling: check for poisoned shadow of
8094 // asm() arguments, then unpoison the result and all the memory locations
8095 // pointed to by those arguments.
8096 // An inline asm() statement in C++ contains lists of input and output
8097 // arguments used by the assembly code. These are mapped to operands of the
8098 // CallInst as follows:
8099 // - nR register outputs ("=r) are returned by value in a single structure
8100 // (SSA value of the CallInst);
8101 // - nO other outputs ("=m" and others) are returned by pointer as first
8102 // nO operands of the CallInst;
8103 // - nI inputs ("r", "m" and others) are passed to CallInst as the
8104 // remaining nI operands.
8105 // The total number of asm() arguments in the source is nR+nO+nI, and the
8106 // corresponding CallInst has nO+nI+1 operands (the last operand is the
8107 // function to be called).
8108 const DataLayout &DL = F.getDataLayout();
8109 CallBase *CB = cast<CallBase>(Val: &I);
8110 IRBuilder<> IRB(&I);
8111 InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8112 int OutputArgs = getNumOutputArgs(IA, CB);
8113 // The last operand of a CallInst is the function itself.
8114 int NumOperands = CB->getNumOperands() - 1;
8115
8116 // Check input arguments. Doing so before unpoisoning output arguments, so
8117 // that we won't overwrite uninit values before checking them.
8118 for (int i = OutputArgs; i < NumOperands; i++) {
8119 Value *Operand = CB->getOperand(i_nocapture: i);
8120 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8121 /*isOutput*/ false);
8122 }
8123 // Unpoison output arguments. This must happen before the actual InlineAsm
8124 // call, so that the shadow for memory published in the asm() statement
8125 // remains valid.
8126 for (int i = 0; i < OutputArgs; i++) {
8127 Value *Operand = CB->getOperand(i_nocapture: i);
8128 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8129 /*isOutput*/ true);
8130 }
8131
8132 setShadow(V: &I, SV: getCleanShadow(V: &I));
8133 setOrigin(V: &I, Origin: getCleanOrigin());
8134 }
8135
8136 void visitFreezeInst(FreezeInst &I) {
8137 // Freeze always returns a fully defined value.
8138 setShadow(V: &I, SV: getCleanShadow(V: &I));
8139 setOrigin(V: &I, Origin: getCleanOrigin());
8140 }
8141
8142 void visitInstruction(Instruction &I) {
8143 // Everything else: stop propagating and check for poisoned shadow.
8144 if (ClDumpStrictInstructions)
8145 dumpInst(I, Prefix: "Strict");
8146 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
8147 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
8148 Value *Operand = I.getOperand(i);
8149 if (Operand->getType()->isSized())
8150 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8151 }
8152 setShadow(V: &I, SV: getCleanShadow(V: &I));
8153 setOrigin(V: &I, Origin: getCleanOrigin());
8154 }
8155};
8156
8157struct VarArgHelperBase : public VarArgHelper {
8158 Function &F;
8159 MemorySanitizer &MS;
8160 MemorySanitizerVisitor &MSV;
8161 SmallVector<CallInst *, 16> VAStartInstrumentationList;
8162 const unsigned VAListTagSize;
8163
8164 VarArgHelperBase(Function &F, MemorySanitizer &MS,
8165 MemorySanitizerVisitor &MSV, unsigned VAListTagSize)
8166 : F(F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
8167
8168 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8169 Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
8170 return IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
8171 }
8172
8173 /// Compute the shadow address for a given va_arg.
8174 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8175 return IRB.CreatePtrAdd(
8176 Ptr: MS.VAArgTLS, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg_va_s");
8177 }
8178
8179 /// Compute the shadow address for a given va_arg.
8180 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset,
8181 unsigned ArgSize) {
8182 // Make sure we don't overflow __msan_va_arg_tls.
8183 if (ArgOffset + ArgSize > kParamTLSSize)
8184 return nullptr;
8185 return getShadowPtrForVAArgument(IRB, ArgOffset);
8186 }
8187
8188 /// Compute the origin address for a given va_arg.
8189 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
8190 // getOriginPtrForVAArgument() is always called after
8191 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
8192 // overflow.
8193 return IRB.CreatePtrAdd(Ptr: MS.VAArgOriginTLS,
8194 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
8195 Name: "_msarg_va_o");
8196 }
8197
8198 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
8199 unsigned BaseOffset) {
8200 // The tails of __msan_va_arg_tls is not large enough to fit full
8201 // value shadow, but it will be copied to backup anyway. Make it
8202 // clean.
8203 if (BaseOffset >= kParamTLSSize)
8204 return;
8205 Value *TailSize =
8206 ConstantInt::getSigned(Ty: IRB.getInt32Ty(), V: kParamTLSSize - BaseOffset);
8207 IRB.CreateMemSet(Ptr: ShadowBase, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8208 Size: TailSize, Align: Align(8));
8209 }
8210
8211 void unpoisonVAListTagForInst(IntrinsicInst &I) {
8212 IRBuilder<> IRB(&I);
8213 Value *VAListTag = I.getArgOperand(i: 0);
8214 const Align Alignment = Align(8);
8215 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
8216 Addr: VAListTag, IRB, ShadowTy: IRB.getInt8Ty(), Alignment, /*isStore*/ true);
8217 // Unpoison the whole __va_list_tag.
8218 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8219 Size: VAListTagSize, Align: Alignment, isVolatile: false);
8220 }
8221
8222 void visitVAStartInst(VAStartInst &I) override {
8223 if (F.getCallingConv() == CallingConv::Win64)
8224 return;
8225 VAStartInstrumentationList.push_back(Elt: &I);
8226 unpoisonVAListTagForInst(I);
8227 }
8228
8229 void visitVACopyInst(VACopyInst &I) override {
8230 if (F.getCallingConv() == CallingConv::Win64)
8231 return;
8232 unpoisonVAListTagForInst(I);
8233 }
8234};
8235
8236/// AMD64-specific implementation of VarArgHelper.
8237struct VarArgAMD64Helper : public VarArgHelperBase {
8238 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
8239 // See a comment in visitCallBase for more details.
8240 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
8241 static const unsigned AMD64FpEndOffsetSSE = 176;
8242 // If SSE is disabled, fp_offset in va_list is zero.
8243 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
8244
8245 unsigned AMD64FpEndOffset;
8246 AllocaInst *VAArgTLSCopy = nullptr;
8247 AllocaInst *VAArgTLSOriginCopy = nullptr;
8248 Value *VAArgOverflowSize = nullptr;
8249
8250 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8251
8252 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
8253 MemorySanitizerVisitor &MSV)
8254 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/24) {
8255 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
8256 for (const auto &Attr : F.getAttributes().getFnAttrs()) {
8257 if (Attr.isStringAttribute() &&
8258 (Attr.getKindAsString() == "target-features")) {
8259 if (Attr.getValueAsString().contains(Other: "-sse"))
8260 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
8261 break;
8262 }
8263 }
8264 }
8265
8266 ArgKind classifyArgument(Value *arg) {
8267 // A very rough approximation of X86_64 argument classification rules.
8268 Type *T = arg->getType();
8269 if (T->isX86_FP80Ty())
8270 return AK_Memory;
8271 if (T->isFPOrFPVectorTy())
8272 return AK_FloatingPoint;
8273 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
8274 return AK_GeneralPurpose;
8275 if (T->isPointerTy())
8276 return AK_GeneralPurpose;
8277 return AK_Memory;
8278 }
8279
8280 // For VarArg functions, store the argument shadow in an ABI-specific format
8281 // that corresponds to va_list layout.
8282 // We do this because Clang lowers va_arg in the frontend, and this pass
8283 // only sees the low level code that deals with va_list internals.
8284 // A much easier alternative (provided that Clang emits va_arg instructions)
8285 // would have been to associate each live instance of va_list with a copy of
8286 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
8287 // order.
8288 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8289 unsigned GpOffset = 0;
8290 unsigned FpOffset = AMD64GpEndOffset;
8291 unsigned OverflowOffset = AMD64FpEndOffset;
8292 const DataLayout &DL = F.getDataLayout();
8293
8294 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8295 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8296 bool IsByVal = CB.paramHasAttr(ArgNo, Kind: Attribute::ByVal);
8297 if (IsByVal) {
8298 // ByVal arguments always go to the overflow area.
8299 // Fixed arguments passed through the overflow area will be stepped
8300 // over by va_start, so don't count them towards the offset.
8301 if (IsFixed)
8302 continue;
8303 assert(A->getType()->isPointerTy());
8304 Type *RealTy = CB.getParamByValType(ArgNo);
8305 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8306 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8307 unsigned BaseOffset = OverflowOffset;
8308 Value *ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8309 Value *OriginBase = nullptr;
8310 if (MS.TrackOrigins)
8311 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8312 OverflowOffset += AlignedSize;
8313
8314 if (OverflowOffset > kParamTLSSize) {
8315 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8316 continue; // We have no space to copy shadow there.
8317 }
8318
8319 Value *ShadowPtr, *OriginPtr;
8320 std::tie(args&: ShadowPtr, args&: OriginPtr) =
8321 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: kShadowTLSAlignment,
8322 /*isStore*/ false);
8323 IRB.CreateMemCpy(Dst: ShadowBase, DstAlign: kShadowTLSAlignment, Src: ShadowPtr,
8324 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8325 if (MS.TrackOrigins)
8326 IRB.CreateMemCpy(Dst: OriginBase, DstAlign: kShadowTLSAlignment, Src: OriginPtr,
8327 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8328 } else {
8329 ArgKind AK = classifyArgument(arg: A);
8330 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
8331 AK = AK_Memory;
8332 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
8333 AK = AK_Memory;
8334 Value *ShadowBase, *OriginBase = nullptr;
8335 switch (AK) {
8336 case AK_GeneralPurpose:
8337 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: GpOffset);
8338 if (MS.TrackOrigins)
8339 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset);
8340 GpOffset += 8;
8341 assert(GpOffset <= kParamTLSSize);
8342 break;
8343 case AK_FloatingPoint:
8344 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: FpOffset);
8345 if (MS.TrackOrigins)
8346 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
8347 FpOffset += 16;
8348 assert(FpOffset <= kParamTLSSize);
8349 break;
8350 case AK_Memory:
8351 if (IsFixed)
8352 continue;
8353 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8354 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8355 unsigned BaseOffset = OverflowOffset;
8356 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8357 if (MS.TrackOrigins) {
8358 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8359 }
8360 OverflowOffset += AlignedSize;
8361 if (OverflowOffset > kParamTLSSize) {
8362 // We have no space to copy shadow there.
8363 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8364 continue;
8365 }
8366 }
8367 // Take fixed arguments into account for GpOffset and FpOffset,
8368 // but don't actually store shadows for them.
8369 // TODO(glider): don't call get*PtrForVAArgument() for them.
8370 if (IsFixed)
8371 continue;
8372 Value *Shadow = MSV.getShadow(V: A);
8373 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowBase, Align: kShadowTLSAlignment);
8374 if (MS.TrackOrigins) {
8375 Value *Origin = MSV.getOrigin(V: A);
8376 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
8377 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
8378 Alignment: std::max(a: kShadowTLSAlignment, b: kMinOriginAlignment));
8379 }
8380 }
8381 }
8382 Constant *OverflowSize =
8383 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AMD64FpEndOffset);
8384 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8385 }
8386
8387 void finalizeInstrumentation() override {
8388 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8389 "finalizeInstrumentation called twice");
8390 if (!VAStartInstrumentationList.empty()) {
8391 // If there is a va_start in this function, make a backup copy of
8392 // va_arg_tls somewhere in the function entry block.
8393 IRBuilder<> IRB(MSV.FnPrologueEnd);
8394 VAArgOverflowSize =
8395 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8396 Value *CopySize = IRB.CreateAdd(
8397 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AMD64FpEndOffset), RHS: VAArgOverflowSize);
8398 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8399 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8400 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8401 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8402
8403 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8404 ID: Intrinsic::umin, LHS: CopySize,
8405 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8406 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8407 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8408 if (MS.TrackOrigins) {
8409 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8410 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
8411 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
8412 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8413 }
8414 }
8415
8416 // Instrument va_start.
8417 // Copy va_list shadow from the backup copy of the TLS contents.
8418 for (CallInst *OrigInst : VAStartInstrumentationList) {
8419 NextNodeIRBuilder IRB(OrigInst);
8420 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8421
8422 Value *RegSaveAreaPtrPtr =
8423 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 16));
8424 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8425 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8426 const Align Alignment = Align(16);
8427 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8428 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8429 Alignment, /*isStore*/ true);
8430 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8431 Size: AMD64FpEndOffset);
8432 if (MS.TrackOrigins)
8433 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
8434 SrcAlign: Alignment, Size: AMD64FpEndOffset);
8435 Value *OverflowArgAreaPtrPtr =
8436 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
8437 Value *OverflowArgAreaPtr =
8438 IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
8439 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
8440 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
8441 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8442 Alignment, /*isStore*/ true);
8443 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
8444 Idx0: AMD64FpEndOffset);
8445 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8446 Size: VAArgOverflowSize);
8447 if (MS.TrackOrigins) {
8448 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
8449 Idx0: AMD64FpEndOffset);
8450 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8451 Size: VAArgOverflowSize);
8452 }
8453 }
8454 }
8455};
8456
8457/// AArch64-specific implementation of VarArgHelper.
8458struct VarArgAArch64Helper : public VarArgHelperBase {
8459 static const unsigned kAArch64GrArgSize = 64;
8460 static const unsigned kAArch64VrArgSize = 128;
8461
8462 static const unsigned AArch64GrBegOffset = 0;
8463 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
8464 // Make VR space aligned to 16 bytes.
8465 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
8466 static const unsigned AArch64VrEndOffset =
8467 AArch64VrBegOffset + kAArch64VrArgSize;
8468 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
8469
8470 AllocaInst *VAArgTLSCopy = nullptr;
8471 Value *VAArgOverflowSize = nullptr;
8472
8473 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8474
8475 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
8476 MemorySanitizerVisitor &MSV)
8477 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/32) {}
8478
8479 // A very rough approximation of aarch64 argument classification rules.
8480 std::pair<ArgKind, uint64_t> classifyArgument(Type *T) {
8481 if (T->isIntOrPtrTy() && T->getPrimitiveSizeInBits() <= 64)
8482 return {AK_GeneralPurpose, 1};
8483 if (T->isFloatingPointTy() && T->getPrimitiveSizeInBits() <= 128)
8484 return {AK_FloatingPoint, 1};
8485
8486 if (T->isArrayTy()) {
8487 auto R = classifyArgument(T: T->getArrayElementType());
8488 R.second *= T->getScalarType()->getArrayNumElements();
8489 return R;
8490 }
8491
8492 if (const FixedVectorType *FV = dyn_cast<FixedVectorType>(Val: T)) {
8493 auto R = classifyArgument(T: FV->getScalarType());
8494 R.second *= FV->getNumElements();
8495 return R;
8496 }
8497
8498 LLVM_DEBUG(errs() << "Unknown vararg type: " << *T << "\n");
8499 return {AK_Memory, 0};
8500 }
8501
8502 // The instrumentation stores the argument shadow in a non ABI-specific
8503 // format because it does not know which argument is named (since Clang,
8504 // like x86_64 case, lowers the va_args in the frontend and this pass only
8505 // sees the low level code that deals with va_list internals).
8506 // The first seven GR registers are saved in the first 56 bytes of the
8507 // va_arg tls arra, followed by the first 8 FP/SIMD registers, and then
8508 // the remaining arguments.
8509 // Using constant offset within the va_arg TLS array allows fast copy
8510 // in the finalize instrumentation.
8511 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8512 unsigned GrOffset = AArch64GrBegOffset;
8513 unsigned VrOffset = AArch64VrBegOffset;
8514 unsigned OverflowOffset = AArch64VAEndOffset;
8515
8516 const DataLayout &DL = F.getDataLayout();
8517 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8518 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8519 auto [AK, RegNum] = classifyArgument(T: A->getType());
8520 if (AK == AK_GeneralPurpose &&
8521 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
8522 AK = AK_Memory;
8523 if (AK == AK_FloatingPoint &&
8524 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
8525 AK = AK_Memory;
8526 Value *Base;
8527 switch (AK) {
8528 case AK_GeneralPurpose:
8529 Base = getShadowPtrForVAArgument(IRB, ArgOffset: GrOffset);
8530 GrOffset += 8 * RegNum;
8531 break;
8532 case AK_FloatingPoint:
8533 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VrOffset);
8534 VrOffset += 16 * RegNum;
8535 break;
8536 case AK_Memory:
8537 // Don't count fixed arguments in the overflow area - va_start will
8538 // skip right over them.
8539 if (IsFixed)
8540 continue;
8541 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8542 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8543 unsigned BaseOffset = OverflowOffset;
8544 Base = getShadowPtrForVAArgument(IRB, ArgOffset: BaseOffset);
8545 OverflowOffset += AlignedSize;
8546 if (OverflowOffset > kParamTLSSize) {
8547 // We have no space to copy shadow there.
8548 CleanUnusedTLS(IRB, ShadowBase: Base, BaseOffset);
8549 continue;
8550 }
8551 break;
8552 }
8553 // Count Gp/Vr fixed arguments to their respective offsets, but don't
8554 // bother to actually store a shadow.
8555 if (IsFixed)
8556 continue;
8557 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8558 }
8559 Constant *OverflowSize =
8560 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AArch64VAEndOffset);
8561 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8562 }
8563
8564 // Retrieve a va_list field of 'void*' size.
8565 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8566 Value *SaveAreaPtrPtr =
8567 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8568 return IRB.CreateLoad(Ty: Type::getInt64Ty(C&: *MS.C), Ptr: SaveAreaPtrPtr);
8569 }
8570
8571 // Retrieve a va_list field of 'int' size.
8572 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8573 Value *SaveAreaPtr =
8574 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8575 Value *SaveArea32 = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: SaveAreaPtr);
8576 return IRB.CreateSExt(V: SaveArea32, DestTy: MS.IntptrTy);
8577 }
8578
8579 void finalizeInstrumentation() override {
8580 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8581 "finalizeInstrumentation called twice");
8582 if (!VAStartInstrumentationList.empty()) {
8583 // If there is a va_start in this function, make a backup copy of
8584 // va_arg_tls somewhere in the function entry block.
8585 IRBuilder<> IRB(MSV.FnPrologueEnd);
8586 VAArgOverflowSize =
8587 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8588 Value *CopySize = IRB.CreateAdd(
8589 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AArch64VAEndOffset), RHS: VAArgOverflowSize);
8590 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8591 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8592 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8593 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8594
8595 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8596 ID: Intrinsic::umin, LHS: CopySize,
8597 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8598 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8599 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8600 }
8601
8602 Value *GrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64GrArgSize);
8603 Value *VrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64VrArgSize);
8604
8605 // Instrument va_start, copy va_list shadow from the backup copy of
8606 // the TLS contents.
8607 for (CallInst *OrigInst : VAStartInstrumentationList) {
8608 NextNodeIRBuilder IRB(OrigInst);
8609
8610 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8611
8612 // The variadic ABI for AArch64 creates two areas to save the incoming
8613 // argument registers (one for 64-bit general register xn-x7 and another
8614 // for 128-bit FP/SIMD vn-v7).
8615 // We need then to propagate the shadow arguments on both regions
8616 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
8617 // The remaining arguments are saved on shadow for 'va::stack'.
8618 // One caveat is it requires only to propagate the non-named arguments,
8619 // however on the call site instrumentation 'all' the arguments are
8620 // saved. So to copy the shadow values from the va_arg TLS array
8621 // we need to adjust the offset for both GR and VR fields based on
8622 // the __{gr,vr}_offs value (since they are stores based on incoming
8623 // named arguments).
8624 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
8625
8626 // Read the stack pointer from the va_list.
8627 Value *StackSaveAreaPtr =
8628 IRB.CreateIntToPtr(V: getVAField64(IRB, VAListTag, offset: 0), DestTy: RegSaveAreaPtrTy);
8629
8630 // Read both the __gr_top and __gr_off and add them up.
8631 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 8);
8632 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, offset: 24);
8633
8634 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
8635 V: IRB.CreateAdd(LHS: GrTopSaveAreaPtr, RHS: GrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8636
8637 // Read both the __vr_top and __vr_off and add them up.
8638 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 16);
8639 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, offset: 28);
8640
8641 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
8642 V: IRB.CreateAdd(LHS: VrTopSaveAreaPtr, RHS: VrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8643
8644 // It does not know how many named arguments is being used and, on the
8645 // callsite all the arguments were saved. Since __gr_off is defined as
8646 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
8647 // argument by ignoring the bytes of shadow from named arguments.
8648 Value *GrRegSaveAreaShadowPtrOff =
8649 IRB.CreateAdd(LHS: GrArgSize, RHS: GrOffSaveArea);
8650
8651 Value *GrRegSaveAreaShadowPtr =
8652 MSV.getShadowOriginPtr(Addr: GrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8653 Alignment: Align(8), /*isStore*/ true)
8654 .first;
8655
8656 Value *GrSrcPtr =
8657 IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy, Offset: GrRegSaveAreaShadowPtrOff);
8658 Value *GrCopySize = IRB.CreateSub(LHS: GrArgSize, RHS: GrRegSaveAreaShadowPtrOff);
8659
8660 IRB.CreateMemCpy(Dst: GrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: GrSrcPtr, SrcAlign: Align(8),
8661 Size: GrCopySize);
8662
8663 // Again, but for FP/SIMD values.
8664 Value *VrRegSaveAreaShadowPtrOff =
8665 IRB.CreateAdd(LHS: VrArgSize, RHS: VrOffSaveArea);
8666
8667 Value *VrRegSaveAreaShadowPtr =
8668 MSV.getShadowOriginPtr(Addr: VrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8669 Alignment: Align(8), /*isStore*/ true)
8670 .first;
8671
8672 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
8673 Ptr: IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy,
8674 Offset: IRB.getInt32(C: AArch64VrBegOffset)),
8675 Offset: VrRegSaveAreaShadowPtrOff);
8676 Value *VrCopySize = IRB.CreateSub(LHS: VrArgSize, RHS: VrRegSaveAreaShadowPtrOff);
8677
8678 IRB.CreateMemCpy(Dst: VrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: VrSrcPtr, SrcAlign: Align(8),
8679 Size: VrCopySize);
8680
8681 // And finally for remaining arguments.
8682 Value *StackSaveAreaShadowPtr =
8683 MSV.getShadowOriginPtr(Addr: StackSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8684 Alignment: Align(16), /*isStore*/ true)
8685 .first;
8686
8687 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
8688 Ptr: VAArgTLSCopy, Offset: IRB.getInt32(C: AArch64VAEndOffset));
8689
8690 IRB.CreateMemCpy(Dst: StackSaveAreaShadowPtr, DstAlign: Align(16), Src: StackSrcPtr,
8691 SrcAlign: Align(16), Size: VAArgOverflowSize);
8692 }
8693 }
8694};
8695
8696/// PowerPC64-specific implementation of VarArgHelper.
8697struct VarArgPowerPC64Helper : public VarArgHelperBase {
8698 AllocaInst *VAArgTLSCopy = nullptr;
8699 Value *VAArgSize = nullptr;
8700
8701 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
8702 MemorySanitizerVisitor &MSV)
8703 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
8704
8705 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8706 // For PowerPC, we need to deal with alignment of stack arguments -
8707 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
8708 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
8709 // For that reason, we compute current offset from stack pointer (which is
8710 // always properly aligned), and offset for the first vararg, then subtract
8711 // them.
8712 unsigned VAArgBase;
8713 Triple TargetTriple(F.getParent()->getTargetTriple());
8714 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
8715 // and 32 bytes for ABIv2. This is usually determined by target
8716 // endianness, but in theory could be overridden by function attribute.
8717 if (TargetTriple.isPPC64ELFv2ABI())
8718 VAArgBase = 32;
8719 else
8720 VAArgBase = 48;
8721 unsigned VAArgOffset = VAArgBase;
8722 const DataLayout &DL = F.getDataLayout();
8723 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8724 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8725 bool IsByVal = CB.paramHasAttr(ArgNo, Kind: Attribute::ByVal);
8726 if (IsByVal) {
8727 assert(A->getType()->isPointerTy());
8728 Type *RealTy = CB.getParamByValType(ArgNo);
8729 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8730 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(8));
8731 if (ArgAlign < 8)
8732 ArgAlign = Align(8);
8733 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8734 if (!IsFixed) {
8735 Value *Base =
8736 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8737 if (Base) {
8738 Value *AShadowPtr, *AOriginPtr;
8739 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8740 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8741 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8742
8743 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8744 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8745 }
8746 }
8747 VAArgOffset += alignTo(Size: ArgSize, A: Align(8));
8748 } else {
8749 Value *Base;
8750 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8751 Align ArgAlign = Align(8);
8752 if (A->getType()->isArrayTy()) {
8753 // Arrays are aligned to element size, except for long double
8754 // arrays, which are aligned to 8 bytes.
8755 Type *ElementTy = A->getType()->getArrayElementType();
8756 if (!ElementTy->isPPC_FP128Ty())
8757 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8758 } else if (A->getType()->isVectorTy()) {
8759 // Vectors are naturally aligned.
8760 ArgAlign = Align(ArgSize);
8761 }
8762 if (ArgAlign < 8)
8763 ArgAlign = Align(8);
8764 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8765 if (DL.isBigEndian()) {
8766 // Adjusting the shadow for argument with size < 8 to match the
8767 // placement of bits in big endian system
8768 if (ArgSize < 8)
8769 VAArgOffset += (8 - ArgSize);
8770 }
8771 if (!IsFixed) {
8772 Base =
8773 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8774 if (Base)
8775 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8776 }
8777 VAArgOffset += ArgSize;
8778 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(8));
8779 }
8780 if (IsFixed)
8781 VAArgBase = VAArgOffset;
8782 }
8783
8784 Constant *TotalVAArgSize =
8785 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8786 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8787 // a new class member i.e. it is the total size of all VarArgs.
8788 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8789 }
8790
8791 void finalizeInstrumentation() override {
8792 assert(!VAArgSize && !VAArgTLSCopy &&
8793 "finalizeInstrumentation called twice");
8794 IRBuilder<> IRB(MSV.FnPrologueEnd);
8795 VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8796 Value *CopySize = VAArgSize;
8797
8798 if (!VAStartInstrumentationList.empty()) {
8799 // If there is a va_start in this function, make a backup copy of
8800 // va_arg_tls somewhere in the function entry block.
8801
8802 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8803 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8804 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8805 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8806
8807 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8808 ID: Intrinsic::umin, LHS: CopySize,
8809 RHS: ConstantInt::get(Ty: IRB.getInt64Ty(), V: kParamTLSSize));
8810 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8811 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8812 }
8813
8814 // Instrument va_start.
8815 // Copy va_list shadow from the backup copy of the TLS contents.
8816 for (CallInst *OrigInst : VAStartInstrumentationList) {
8817 NextNodeIRBuilder IRB(OrigInst);
8818 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8819 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8820
8821 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
8822
8823 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8824 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8825 const DataLayout &DL = F.getDataLayout();
8826 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8827 const Align Alignment = Align(IntptrSize);
8828 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8829 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8830 Alignment, /*isStore*/ true);
8831 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8832 Size: CopySize);
8833 }
8834 }
8835};
8836
8837/// PowerPC32-specific implementation of VarArgHelper.
8838struct VarArgPowerPC32Helper : public VarArgHelperBase {
8839 AllocaInst *VAArgTLSCopy = nullptr;
8840 Value *VAArgSize = nullptr;
8841
8842 VarArgPowerPC32Helper(Function &F, MemorySanitizer &MS,
8843 MemorySanitizerVisitor &MSV)
8844 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/12) {}
8845
8846 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8847 unsigned VAArgBase;
8848 // Parameter save area is 8 bytes from frame pointer in PPC32
8849 VAArgBase = 8;
8850 unsigned VAArgOffset = VAArgBase;
8851 const DataLayout &DL = F.getDataLayout();
8852 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8853 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8854 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8855 bool IsByVal = CB.paramHasAttr(ArgNo, Kind: Attribute::ByVal);
8856 if (IsByVal) {
8857 assert(A->getType()->isPointerTy());
8858 Type *RealTy = CB.getParamByValType(ArgNo);
8859 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8860 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
8861 if (ArgAlign < IntptrSize)
8862 ArgAlign = Align(IntptrSize);
8863 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8864 if (!IsFixed) {
8865 Value *Base =
8866 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8867 if (Base) {
8868 Value *AShadowPtr, *AOriginPtr;
8869 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8870 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8871 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8872
8873 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8874 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8875 }
8876 }
8877 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
8878 } else {
8879 Value *Base;
8880 Type *ArgTy = A->getType();
8881
8882 // On PPC 32 floating point variable arguments are stored in separate
8883 // area: fp_save_area = reg_save_area + 4*8. We do not copy shaodow for
8884 // them as they will be found when checking call arguments.
8885 if (!ArgTy->isFloatingPointTy()) {
8886 uint64_t ArgSize = DL.getTypeAllocSize(Ty: ArgTy);
8887 Align ArgAlign = Align(IntptrSize);
8888 if (ArgTy->isArrayTy()) {
8889 // Arrays are aligned to element size, except for long double
8890 // arrays, which are aligned to 8 bytes.
8891 Type *ElementTy = ArgTy->getArrayElementType();
8892 if (!ElementTy->isPPC_FP128Ty())
8893 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8894 } else if (ArgTy->isVectorTy()) {
8895 // Vectors are naturally aligned.
8896 ArgAlign = Align(ArgSize);
8897 }
8898 if (ArgAlign < IntptrSize)
8899 ArgAlign = Align(IntptrSize);
8900 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8901 if (DL.isBigEndian()) {
8902 // Adjusting the shadow for argument with size < IntptrSize to match
8903 // the placement of bits in big endian system
8904 if (ArgSize < IntptrSize)
8905 VAArgOffset += (IntptrSize - ArgSize);
8906 }
8907 if (!IsFixed) {
8908 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase,
8909 ArgSize);
8910 if (Base)
8911 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base,
8912 Align: kShadowTLSAlignment);
8913 }
8914 VAArgOffset += ArgSize;
8915 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
8916 }
8917 }
8918 }
8919
8920 Constant *TotalVAArgSize =
8921 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8922 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8923 // a new class member i.e. it is the total size of all VarArgs.
8924 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8925 }
8926
8927 void finalizeInstrumentation() override {
8928 assert(!VAArgSize && !VAArgTLSCopy &&
8929 "finalizeInstrumentation called twice");
8930 IRBuilder<> IRB(MSV.FnPrologueEnd);
8931 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
8932 Value *CopySize = VAArgSize;
8933
8934 if (!VAStartInstrumentationList.empty()) {
8935 // If there is a va_start in this function, make a backup copy of
8936 // va_arg_tls somewhere in the function entry block.
8937
8938 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8939 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8940 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8941 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8942
8943 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8944 ID: Intrinsic::umin, LHS: CopySize,
8945 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8946 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8947 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8948 }
8949
8950 // Instrument va_start.
8951 // Copy va_list shadow from the backup copy of the TLS contents.
8952 for (CallInst *OrigInst : VAStartInstrumentationList) {
8953 NextNodeIRBuilder IRB(OrigInst);
8954 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8955 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8956 Value *RegSaveAreaSize = CopySize;
8957
8958 // In PPC32 va_list_tag is a struct
8959 RegSaveAreaPtrPtr =
8960 IRB.CreateAdd(LHS: RegSaveAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
8961
8962 // On PPC 32 reg_save_area can only hold 32 bytes of data
8963 RegSaveAreaSize = IRB.CreateBinaryIntrinsic(
8964 ID: Intrinsic::umin, LHS: CopySize, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
8965
8966 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
8967 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8968
8969 const DataLayout &DL = F.getDataLayout();
8970 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8971 const Align Alignment = Align(IntptrSize);
8972
8973 { // Copy reg save area
8974 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8975 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8976 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8977 Alignment, /*isStore*/ true);
8978 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy,
8979 SrcAlign: Alignment, Size: RegSaveAreaSize);
8980
8981 RegSaveAreaShadowPtr =
8982 IRB.CreatePtrToInt(V: RegSaveAreaShadowPtr, DestTy: MS.IntptrTy);
8983 Value *FPSaveArea = IRB.CreateAdd(LHS: RegSaveAreaShadowPtr,
8984 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
8985 FPSaveArea = IRB.CreateIntToPtr(V: FPSaveArea, DestTy: MS.PtrTy);
8986 // We fill fp shadow with zeroes as uninitialized fp args should have
8987 // been found during call base check
8988 IRB.CreateMemSet(Ptr: FPSaveArea, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8989 Size: ConstantInt::get(Ty: MS.IntptrTy, V: 32), Align: Alignment);
8990 }
8991
8992 { // Copy overflow area
8993 // RegSaveAreaSize is min(CopySize, 32) -> no overflow can occur
8994 Value *OverflowAreaSize = IRB.CreateSub(LHS: CopySize, RHS: RegSaveAreaSize);
8995
8996 Value *OverflowAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8997 OverflowAreaPtrPtr =
8998 IRB.CreateAdd(LHS: OverflowAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 4));
8999 OverflowAreaPtrPtr = IRB.CreateIntToPtr(V: OverflowAreaPtrPtr, DestTy: MS.PtrTy);
9000
9001 Value *OverflowAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowAreaPtrPtr);
9002
9003 Value *OverflowAreaShadowPtr, *OverflowAreaOriginPtr;
9004 std::tie(args&: OverflowAreaShadowPtr, args&: OverflowAreaOriginPtr) =
9005 MSV.getShadowOriginPtr(Addr: OverflowAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9006 Alignment, /*isStore*/ true);
9007
9008 Value *OverflowVAArgTLSCopyPtr =
9009 IRB.CreatePtrToInt(V: VAArgTLSCopy, DestTy: MS.IntptrTy);
9010 OverflowVAArgTLSCopyPtr =
9011 IRB.CreateAdd(LHS: OverflowVAArgTLSCopyPtr, RHS: RegSaveAreaSize);
9012
9013 OverflowVAArgTLSCopyPtr =
9014 IRB.CreateIntToPtr(V: OverflowVAArgTLSCopyPtr, DestTy: MS.PtrTy);
9015 IRB.CreateMemCpy(Dst: OverflowAreaShadowPtr, DstAlign: Alignment,
9016 Src: OverflowVAArgTLSCopyPtr, SrcAlign: Alignment, Size: OverflowAreaSize);
9017 }
9018 }
9019 }
9020};
9021
9022/// SystemZ-specific implementation of VarArgHelper.
9023struct VarArgSystemZHelper : public VarArgHelperBase {
9024 static const unsigned SystemZGpOffset = 16;
9025 static const unsigned SystemZGpEndOffset = 56;
9026 static const unsigned SystemZFpOffset = 128;
9027 static const unsigned SystemZFpEndOffset = 160;
9028 static const unsigned SystemZMaxVrArgs = 8;
9029 static const unsigned SystemZRegSaveAreaSize = 160;
9030 static const unsigned SystemZOverflowOffset = 160;
9031 static const unsigned SystemZVAListTagSize = 32;
9032 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
9033 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
9034
9035 bool IsSoftFloatABI;
9036 AllocaInst *VAArgTLSCopy = nullptr;
9037 AllocaInst *VAArgTLSOriginCopy = nullptr;
9038 Value *VAArgOverflowSize = nullptr;
9039
9040 enum class ArgKind {
9041 GeneralPurpose,
9042 FloatingPoint,
9043 Vector,
9044 Memory,
9045 Indirect,
9046 };
9047
9048 enum class ShadowExtension { None, Zero, Sign };
9049
9050 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
9051 MemorySanitizerVisitor &MSV)
9052 : VarArgHelperBase(F, MS, MSV, SystemZVAListTagSize),
9053 IsSoftFloatABI(F.getFnAttribute(Kind: "use-soft-float").getValueAsBool()) {}
9054
9055 ArgKind classifyArgument(Type *T) {
9056 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
9057 // only a few possibilities of what it can be. In particular, enums, single
9058 // element structs and large types have already been taken care of.
9059
9060 // Some i128 and fp128 arguments are converted to pointers only in the
9061 // back end.
9062 if (T->isIntegerTy(BitWidth: 128) || T->isFP128Ty())
9063 return ArgKind::Indirect;
9064 if (T->isFloatingPointTy())
9065 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
9066 if (T->isIntegerTy() || T->isPointerTy())
9067 return ArgKind::GeneralPurpose;
9068 if (T->isVectorTy())
9069 return ArgKind::Vector;
9070 return ArgKind::Memory;
9071 }
9072
9073 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
9074 // ABI says: "One of the simple integer types no more than 64 bits wide.
9075 // ... If such an argument is shorter than 64 bits, replace it by a full
9076 // 64-bit integer representing the same number, using sign or zero
9077 // extension". Shadow for an integer argument has the same type as the
9078 // argument itself, so it can be sign or zero extended as well.
9079 bool ZExt = CB.paramHasAttr(ArgNo, Kind: Attribute::ZExt);
9080 bool SExt = CB.paramHasAttr(ArgNo, Kind: Attribute::SExt);
9081 if (ZExt) {
9082 assert(!SExt);
9083 return ShadowExtension::Zero;
9084 }
9085 if (SExt) {
9086 assert(!ZExt);
9087 return ShadowExtension::Sign;
9088 }
9089 return ShadowExtension::None;
9090 }
9091
9092 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9093 unsigned GpOffset = SystemZGpOffset;
9094 unsigned FpOffset = SystemZFpOffset;
9095 unsigned VrIndex = 0;
9096 unsigned OverflowOffset = SystemZOverflowOffset;
9097 const DataLayout &DL = F.getDataLayout();
9098 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9099 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9100 // SystemZABIInfo does not produce ByVal parameters.
9101 assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal));
9102 Type *T = A->getType();
9103 ArgKind AK = classifyArgument(T);
9104 if (AK == ArgKind::Indirect) {
9105 T = MS.PtrTy;
9106 AK = ArgKind::GeneralPurpose;
9107 }
9108 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
9109 AK = ArgKind::Memory;
9110 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
9111 AK = ArgKind::Memory;
9112 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
9113 AK = ArgKind::Memory;
9114 Value *ShadowBase = nullptr;
9115 Value *OriginBase = nullptr;
9116 ShadowExtension SE = ShadowExtension::None;
9117 switch (AK) {
9118 case ArgKind::GeneralPurpose: {
9119 // Always keep track of GpOffset, but store shadow only for varargs.
9120 uint64_t ArgSize = 8;
9121 if (GpOffset + ArgSize <= kParamTLSSize) {
9122 if (!IsFixed) {
9123 SE = getShadowExtension(CB, ArgNo);
9124 uint64_t GapSize = 0;
9125 if (SE == ShadowExtension::None) {
9126 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9127 assert(ArgAllocSize <= ArgSize);
9128 GapSize = ArgSize - ArgAllocSize;
9129 }
9130 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9131 if (MS.TrackOrigins)
9132 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9133 }
9134 GpOffset += ArgSize;
9135 } else {
9136 GpOffset = kParamTLSSize;
9137 }
9138 break;
9139 }
9140 case ArgKind::FloatingPoint: {
9141 // Always keep track of FpOffset, but store shadow only for varargs.
9142 uint64_t ArgSize = 8;
9143 if (FpOffset + ArgSize <= kParamTLSSize) {
9144 if (!IsFixed) {
9145 // PoP says: "A short floating-point datum requires only the
9146 // left-most 32 bit positions of a floating-point register".
9147 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
9148 // don't extend shadow and don't mind the gap.
9149 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: FpOffset);
9150 if (MS.TrackOrigins)
9151 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
9152 }
9153 FpOffset += ArgSize;
9154 } else {
9155 FpOffset = kParamTLSSize;
9156 }
9157 break;
9158 }
9159 case ArgKind::Vector: {
9160 // Keep track of VrIndex. No need to store shadow, since vector varargs
9161 // go through AK_Memory.
9162 assert(IsFixed);
9163 VrIndex++;
9164 break;
9165 }
9166 case ArgKind::Memory: {
9167 // Keep track of OverflowOffset and store shadow only for varargs.
9168 // Ignore fixed args, since we need to copy only the vararg portion of
9169 // the overflow area shadow.
9170 if (!IsFixed) {
9171 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9172 uint64_t ArgSize = alignTo(Value: ArgAllocSize, Align: 8);
9173 if (OverflowOffset + ArgSize <= kParamTLSSize) {
9174 SE = getShadowExtension(CB, ArgNo);
9175 uint64_t GapSize =
9176 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
9177 ShadowBase =
9178 getShadowAddrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9179 if (MS.TrackOrigins)
9180 OriginBase =
9181 getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9182 OverflowOffset += ArgSize;
9183 } else {
9184 OverflowOffset = kParamTLSSize;
9185 }
9186 }
9187 break;
9188 }
9189 case ArgKind::Indirect:
9190 llvm_unreachable("Indirect must be converted to GeneralPurpose");
9191 }
9192 if (ShadowBase == nullptr)
9193 continue;
9194 Value *Shadow = MSV.getShadow(V: A);
9195 if (SE != ShadowExtension::None)
9196 Shadow = MSV.CreateShadowCast(IRB, V: Shadow, dstTy: IRB.getInt64Ty(),
9197 /*Signed*/ SE == ShadowExtension::Sign);
9198 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: MS.PtrTy, Name: "_msarg_va_s");
9199 IRB.CreateStore(Val: Shadow, Ptr: ShadowBase);
9200 if (MS.TrackOrigins) {
9201 Value *Origin = MSV.getOrigin(V: A);
9202 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
9203 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
9204 Alignment: kMinOriginAlignment);
9205 }
9206 }
9207 Constant *OverflowSize = ConstantInt::get(
9208 Ty: IRB.getInt64Ty(), V: OverflowOffset - SystemZOverflowOffset);
9209 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
9210 }
9211
9212 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
9213 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
9214 V: IRB.CreateAdd(
9215 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9216 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZRegSaveAreaPtrOffset)),
9217 DestTy: MS.PtrTy);
9218 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9219 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9220 const Align Alignment = Align(8);
9221 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9222 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
9223 /*isStore*/ true);
9224 // TODO(iii): copy only fragments filled by visitCallBase()
9225 // TODO(iii): support packed-stack && !use-soft-float
9226 // For use-soft-float functions, it is enough to copy just the GPRs.
9227 unsigned RegSaveAreaSize =
9228 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
9229 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9230 Size: RegSaveAreaSize);
9231 if (MS.TrackOrigins)
9232 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
9233 SrcAlign: Alignment, Size: RegSaveAreaSize);
9234 }
9235
9236 // FIXME: This implementation limits OverflowOffset to kParamTLSSize, so we
9237 // don't know real overflow size and can't clear shadow beyond kParamTLSSize.
9238 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
9239 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
9240 V: IRB.CreateAdd(
9241 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9242 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowArgAreaPtrOffset)),
9243 DestTy: MS.PtrTy);
9244 Value *OverflowArgAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
9245 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
9246 const Align Alignment = Align(8);
9247 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
9248 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9249 Alignment, /*isStore*/ true);
9250 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
9251 Idx0: SystemZOverflowOffset);
9252 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9253 Size: VAArgOverflowSize);
9254 if (MS.TrackOrigins) {
9255 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
9256 Idx0: SystemZOverflowOffset);
9257 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9258 Size: VAArgOverflowSize);
9259 }
9260 }
9261
9262 void finalizeInstrumentation() override {
9263 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
9264 "finalizeInstrumentation called twice");
9265 if (!VAStartInstrumentationList.empty()) {
9266 // If there is a va_start in this function, make a backup copy of
9267 // va_arg_tls somewhere in the function entry block.
9268 IRBuilder<> IRB(MSV.FnPrologueEnd);
9269 VAArgOverflowSize =
9270 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
9271 Value *CopySize =
9272 IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowOffset),
9273 RHS: VAArgOverflowSize);
9274 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9275 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9276 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9277 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9278
9279 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9280 ID: Intrinsic::umin, LHS: CopySize,
9281 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9282 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9283 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9284 if (MS.TrackOrigins) {
9285 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9286 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
9287 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
9288 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9289 }
9290 }
9291
9292 // Instrument va_start.
9293 // Copy va_list shadow from the backup copy of the TLS contents.
9294 for (CallInst *OrigInst : VAStartInstrumentationList) {
9295 NextNodeIRBuilder IRB(OrigInst);
9296 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9297 copyRegSaveArea(IRB, VAListTag);
9298 copyOverflowArea(IRB, VAListTag);
9299 }
9300 }
9301};
9302
9303/// i386-specific implementation of VarArgHelper.
9304struct VarArgI386Helper : public VarArgHelperBase {
9305 AllocaInst *VAArgTLSCopy = nullptr;
9306 Value *VAArgSize = nullptr;
9307
9308 VarArgI386Helper(Function &F, MemorySanitizer &MS,
9309 MemorySanitizerVisitor &MSV)
9310 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/4) {}
9311
9312 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9313 const DataLayout &DL = F.getDataLayout();
9314 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9315 unsigned VAArgOffset = 0;
9316 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9317 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9318 bool IsByVal = CB.paramHasAttr(ArgNo, Kind: Attribute::ByVal);
9319 if (IsByVal) {
9320 assert(A->getType()->isPointerTy());
9321 Type *RealTy = CB.getParamByValType(ArgNo);
9322 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
9323 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
9324 if (ArgAlign < IntptrSize)
9325 ArgAlign = Align(IntptrSize);
9326 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9327 if (!IsFixed) {
9328 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9329 if (Base) {
9330 Value *AShadowPtr, *AOriginPtr;
9331 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
9332 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
9333 Alignment: kShadowTLSAlignment, /*isStore*/ false);
9334
9335 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
9336 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
9337 }
9338 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9339 }
9340 } else {
9341 Value *Base;
9342 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9343 Align ArgAlign = Align(IntptrSize);
9344 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9345 if (DL.isBigEndian()) {
9346 // Adjusting the shadow for argument with size < IntptrSize to match
9347 // the placement of bits in big endian system
9348 if (ArgSize < IntptrSize)
9349 VAArgOffset += (IntptrSize - ArgSize);
9350 }
9351 if (!IsFixed) {
9352 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9353 if (Base)
9354 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9355 VAArgOffset += ArgSize;
9356 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9357 }
9358 }
9359 }
9360
9361 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9362 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9363 // a new class member i.e. it is the total size of all VarArgs.
9364 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9365 }
9366
9367 void finalizeInstrumentation() override {
9368 assert(!VAArgSize && !VAArgTLSCopy &&
9369 "finalizeInstrumentation called twice");
9370 IRBuilder<> IRB(MSV.FnPrologueEnd);
9371 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9372 Value *CopySize = VAArgSize;
9373
9374 if (!VAStartInstrumentationList.empty()) {
9375 // If there is a va_start in this function, make a backup copy of
9376 // va_arg_tls somewhere in the function entry block.
9377 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9378 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9379 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9380 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9381
9382 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9383 ID: Intrinsic::umin, LHS: CopySize,
9384 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9385 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9386 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9387 }
9388
9389 // Instrument va_start.
9390 // Copy va_list shadow from the backup copy of the TLS contents.
9391 for (CallInst *OrigInst : VAStartInstrumentationList) {
9392 NextNodeIRBuilder IRB(OrigInst);
9393 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9394 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9395 Value *RegSaveAreaPtrPtr =
9396 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9397 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9398 Value *RegSaveAreaPtr =
9399 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9400 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9401 const DataLayout &DL = F.getDataLayout();
9402 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9403 const Align Alignment = Align(IntptrSize);
9404 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9405 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9406 Alignment, /*isStore*/ true);
9407 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9408 Size: CopySize);
9409 }
9410 }
9411};
9412
9413/// Implementation of VarArgHelper that is used for ARM32, MIPS, RISCV,
9414/// LoongArch64.
9415struct VarArgGenericHelper : public VarArgHelperBase {
9416 AllocaInst *VAArgTLSCopy = nullptr;
9417 Value *VAArgSize = nullptr;
9418
9419 VarArgGenericHelper(Function &F, MemorySanitizer &MS,
9420 MemorySanitizerVisitor &MSV, const unsigned VAListTagSize)
9421 : VarArgHelperBase(F, MS, MSV, VAListTagSize) {}
9422
9423 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9424 unsigned VAArgOffset = 0;
9425 const DataLayout &DL = F.getDataLayout();
9426 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9427 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9428 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9429 if (IsFixed)
9430 continue;
9431 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9432 if (DL.isBigEndian()) {
9433 // Adjusting the shadow for argument with size < IntptrSize to match the
9434 // placement of bits in big endian system
9435 if (ArgSize < IntptrSize)
9436 VAArgOffset += (IntptrSize - ArgSize);
9437 }
9438 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9439 VAArgOffset += ArgSize;
9440 VAArgOffset = alignTo(Value: VAArgOffset, Align: IntptrSize);
9441 if (!Base)
9442 continue;
9443 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9444 }
9445
9446 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9447 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9448 // a new class member i.e. it is the total size of all VarArgs.
9449 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9450 }
9451
9452 void finalizeInstrumentation() override {
9453 assert(!VAArgSize && !VAArgTLSCopy &&
9454 "finalizeInstrumentation called twice");
9455 IRBuilder<> IRB(MSV.FnPrologueEnd);
9456 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9457 Value *CopySize = VAArgSize;
9458
9459 if (!VAStartInstrumentationList.empty()) {
9460 // If there is a va_start in this function, make a backup copy of
9461 // va_arg_tls somewhere in the function entry block.
9462 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9463 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9464 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9465 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9466
9467 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9468 ID: Intrinsic::umin, LHS: CopySize,
9469 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9470 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9471 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9472 }
9473
9474 // Instrument va_start.
9475 // Copy va_list shadow from the backup copy of the TLS contents.
9476 for (CallInst *OrigInst : VAStartInstrumentationList) {
9477 NextNodeIRBuilder IRB(OrigInst);
9478 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9479 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9480 Value *RegSaveAreaPtrPtr =
9481 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9482 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9483 Value *RegSaveAreaPtr =
9484 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9485 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9486 const DataLayout &DL = F.getDataLayout();
9487 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9488 const Align Alignment = Align(IntptrSize);
9489 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9490 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9491 Alignment, /*isStore*/ true);
9492 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9493 Size: CopySize);
9494 }
9495 }
9496};
9497
9498// ARM32, Loongarch64, MIPS and RISCV share the same calling conventions
9499// regarding VAArgs.
9500using VarArgARM32Helper = VarArgGenericHelper;
9501using VarArgRISCVHelper = VarArgGenericHelper;
9502using VarArgMIPSHelper = VarArgGenericHelper;
9503using VarArgLoongArch64Helper = VarArgGenericHelper;
9504using VarArgHexagonHelper = VarArgGenericHelper;
9505
9506/// A no-op implementation of VarArgHelper.
9507struct VarArgNoOpHelper : public VarArgHelper {
9508 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
9509 MemorySanitizerVisitor &MSV) {}
9510
9511 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
9512
9513 void visitVAStartInst(VAStartInst &I) override {}
9514
9515 void visitVACopyInst(VACopyInst &I) override {}
9516
9517 void finalizeInstrumentation() override {}
9518};
9519
9520} // end anonymous namespace
9521
9522static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
9523 MemorySanitizerVisitor &Visitor) {
9524 // VarArg handling is only implemented on AMD64. False positives are possible
9525 // on other platforms.
9526 Triple TargetTriple(Func.getParent()->getTargetTriple());
9527
9528 if (TargetTriple.getArch() == Triple::x86)
9529 return new VarArgI386Helper(Func, Msan, Visitor);
9530
9531 if (TargetTriple.getArch() == Triple::x86_64)
9532 return new VarArgAMD64Helper(Func, Msan, Visitor);
9533
9534 if (TargetTriple.isARM())
9535 return new VarArgARM32Helper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9536
9537 if (TargetTriple.isAArch64())
9538 return new VarArgAArch64Helper(Func, Msan, Visitor);
9539
9540 if (TargetTriple.isSystemZ())
9541 return new VarArgSystemZHelper(Func, Msan, Visitor);
9542
9543 // On PowerPC32 VAListTag is a struct
9544 // {char, char, i16 padding, char *, char *}
9545 if (TargetTriple.isPPC32())
9546 return new VarArgPowerPC32Helper(Func, Msan, Visitor);
9547
9548 if (TargetTriple.isPPC64())
9549 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
9550
9551 if (TargetTriple.isRISCV32())
9552 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9553
9554 if (TargetTriple.isRISCV64())
9555 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9556
9557 if (TargetTriple.isMIPS32())
9558 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9559
9560 if (TargetTriple.isMIPS64())
9561 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9562
9563 if (TargetTriple.isLoongArch64())
9564 return new VarArgLoongArch64Helper(Func, Msan, Visitor,
9565 /*VAListTagSize=*/8);
9566
9567 if (TargetTriple.getArch() == Triple::hexagon)
9568 return new VarArgHexagonHelper(Func, Msan, Visitor, /*VAListTagSize=*/12);
9569
9570 return new VarArgNoOpHelper(Func, Msan, Visitor);
9571}
9572
9573bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
9574 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
9575 return false;
9576
9577 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
9578 return false;
9579
9580 MemorySanitizerVisitor Visitor(F, *this, TLI);
9581
9582 // Clear out memory attributes.
9583 AttributeMask B;
9584 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
9585 F.removeFnAttrs(Attrs: B);
9586
9587 return Visitor.runOnFunction();
9588}
9589