1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html)
15/// We associate a few shadow bits with every byte of the application memory,
16/// poison the shadow of the malloc-ed or alloca-ed memory, load the shadow,
17/// bits on every memory read, propagate the shadow bits through some of the
18/// arithmetic instruction (including MOV), store the shadow bits on every
19/// memory write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// Note that the sanitizer code has to deal with how shadow/origin pairs
126/// returned by the these functions are represented in different ABIs. In
127/// the X86_64 ABI they are returned in RDX:RAX, in PowerPC64 they are
128/// returned in r3 and r4, and in the SystemZ ABI they are written to memory
129/// pointed to by a hidden parameter.
130/// - TLS variables are stored in a single per-task struct. A call to a
131/// function __msan_get_context_state() returning a pointer to that struct
132/// is inserted into every instrumented function before the entry block;
133/// - __msan_warning() takes a 32-bit origin parameter;
134/// - local variables are poisoned with __msan_poison_alloca() upon function
135/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
136/// function;
137/// - the pass doesn't declare any global variables or add global constructors
138/// to the translation unit.
139///
140/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
141/// calls, making sure we're on the safe side wrt. possible false positives.
142///
143/// KernelMemorySanitizer only supports X86_64, SystemZ and PowerPC64 at the
144/// moment.
145///
146//
147// FIXME: This sanitizer does not yet handle scalable vectors
148//
149//===----------------------------------------------------------------------===//
150
151#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
152#include "InstrumentationOptions.h"
153#include "llvm/ADT/APInt.h"
154#include "llvm/ADT/ArrayRef.h"
155#include "llvm/ADT/DenseMap.h"
156#include "llvm/ADT/DepthFirstIterator.h"
157#include "llvm/ADT/SetVector.h"
158#include "llvm/ADT/SmallPtrSet.h"
159#include "llvm/ADT/SmallVector.h"
160#include "llvm/ADT/StringExtras.h"
161#include "llvm/ADT/StringRef.h"
162#include "llvm/Analysis/GlobalsModRef.h"
163#include "llvm/Analysis/TargetLibraryInfo.h"
164#include "llvm/Analysis/ValueTracking.h"
165#include "llvm/IR/Argument.h"
166#include "llvm/IR/AttributeMask.h"
167#include "llvm/IR/Attributes.h"
168#include "llvm/IR/BasicBlock.h"
169#include "llvm/IR/CallingConv.h"
170#include "llvm/IR/Constant.h"
171#include "llvm/IR/Constants.h"
172#include "llvm/IR/DataLayout.h"
173#include "llvm/IR/DerivedTypes.h"
174#include "llvm/IR/Function.h"
175#include "llvm/IR/GlobalValue.h"
176#include "llvm/IR/GlobalVariable.h"
177#include "llvm/IR/IRBuilder.h"
178#include "llvm/IR/InlineAsm.h"
179#include "llvm/IR/InstVisitor.h"
180#include "llvm/IR/InstrTypes.h"
181#include "llvm/IR/Instruction.h"
182#include "llvm/IR/Instructions.h"
183#include "llvm/IR/IntrinsicInst.h"
184#include "llvm/IR/Intrinsics.h"
185#include "llvm/IR/IntrinsicsAArch64.h"
186#include "llvm/IR/IntrinsicsX86.h"
187#include "llvm/IR/MDBuilder.h"
188#include "llvm/IR/Module.h"
189#include "llvm/IR/Type.h"
190#include "llvm/IR/Value.h"
191#include "llvm/IR/ValueMap.h"
192#include "llvm/Support/Alignment.h"
193#include "llvm/Support/AtomicOrdering.h"
194#include "llvm/Support/Casting.h"
195#include "llvm/Support/Debug.h"
196#include "llvm/Support/DebugCounter.h"
197#include "llvm/Support/ErrorHandling.h"
198#include "llvm/Support/MathExtras.h"
199#include "llvm/Support/raw_ostream.h"
200#include "llvm/TargetParser/Triple.h"
201#include "llvm/Transforms/Utils/BasicBlockUtils.h"
202#include "llvm/Transforms/Utils/Instrumentation.h"
203#include "llvm/Transforms/Utils/Local.h"
204#include "llvm/Transforms/Utils/ModuleUtils.h"
205#include <algorithm>
206#include <cassert>
207#include <cstddef>
208#include <cstdint>
209#include <memory>
210#include <numeric>
211#include <string>
212#include <tuple>
213
214using namespace llvm;
215
216#define DEBUG_TYPE "msan"
217
218DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
219 "Controls which checks to insert");
220
221DEBUG_COUNTER(DebugInstrumentInstruction, "msan-instrument-instruction",
222 "Controls which instruction to instrument");
223
224static const unsigned kOriginSize = 4;
225static const Align kMinOriginAlignment = Align(4);
226static const Align kShadowTLSAlignment = Align(8);
227
228// These constants must be kept in sync with the ones in msan.h.
229// TODO: increase size to match SVE/SVE2/SME/SME2 limits
230static const unsigned kParamTLSSize = 800;
231static const unsigned kRetvalTLSSize = 800;
232
233// Accesses sizes are powers of two: 1, 2, 4, 8.
234static const size_t kNumberOfAccessSizes = 4;
235
236const char kMsanModuleCtorName[] = "msan.module_ctor";
237const char kMsanInitName[] = "__msan_init";
238
239namespace {
240
241// Memory map parameters used in application-to-shadow address calculation.
242// Offset = (Addr & ~AndMask) ^ XorMask
243// Shadow = ShadowBase + Offset
244// Origin = OriginBase + Offset
245struct MemoryMapParams {
246 uint64_t AndMask;
247 uint64_t XorMask;
248 uint64_t ShadowBase;
249 uint64_t OriginBase;
250};
251
252struct PlatformMemoryMapParams {
253 const MemoryMapParams *bits32;
254 const MemoryMapParams *bits64;
255};
256
257} // end anonymous namespace
258
259// i386 Linux
260static const MemoryMapParams Linux_I386_MemoryMapParams = {
261 .AndMask: 0x000080000000, // AndMask
262 .XorMask: 0, // XorMask (not used)
263 .ShadowBase: 0, // ShadowBase (not used)
264 .OriginBase: 0x000040000000, // OriginBase
265};
266
267// x86_64 Linux
268static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
269 .AndMask: 0, // AndMask (not used)
270 .XorMask: 0x500000000000, // XorMask
271 .ShadowBase: 0, // ShadowBase (not used)
272 .OriginBase: 0x100000000000, // OriginBase
273};
274
275// mips32 Linux
276// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
277// after picking good constants
278
279// mips64 Linux
280static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
281 .AndMask: 0, // AndMask (not used)
282 .XorMask: 0x008000000000, // XorMask
283 .ShadowBase: 0, // ShadowBase (not used)
284 .OriginBase: 0x002000000000, // OriginBase
285};
286
287// ppc32 Linux
288// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
289// after picking good constants
290
291// ppc64 Linux
292static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
293 .AndMask: 0xE00000000000, // AndMask
294 .XorMask: 0x100000000000, // XorMask
295 .ShadowBase: 0x080000000000, // ShadowBase
296 .OriginBase: 0x1C0000000000, // OriginBase
297};
298
299// s390x Linux
300static const MemoryMapParams Linux_S390X_MemoryMapParams = {
301 .AndMask: 0xC00000000000, // AndMask
302 .XorMask: 0, // XorMask (not used)
303 .ShadowBase: 0x080000000000, // ShadowBase
304 .OriginBase: 0x1C0000000000, // OriginBase
305};
306
307// arm32 Linux
308// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
309// after picking good constants
310
311// aarch64 Linux
312static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
313 .AndMask: 0, // AndMask (not used)
314 .XorMask: 0x0B00000000000, // XorMask
315 .ShadowBase: 0, // ShadowBase (not used)
316 .OriginBase: 0x0200000000000, // OriginBase
317};
318
319// loongarch64 Linux
320static const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
321 .AndMask: 0, // AndMask (not used)
322 .XorMask: 0x500000000000, // XorMask
323 .ShadowBase: 0, // ShadowBase (not used)
324 .OriginBase: 0x100000000000, // OriginBase
325};
326
327// hexagon Linux
328static const MemoryMapParams Linux_Hexagon_MemoryMapParams = {
329 .AndMask: 0, // AndMask (not used)
330 .XorMask: 0x20000000, // XorMask
331 .ShadowBase: 0, // ShadowBase (not used)
332 .OriginBase: 0x50000000, // OriginBase
333};
334
335// riscv32 Linux
336// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
337// after picking good constants
338
339// aarch64 FreeBSD
340static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
341 .AndMask: 0x1800000000000, // AndMask
342 .XorMask: 0x0400000000000, // XorMask
343 .ShadowBase: 0x0200000000000, // ShadowBase
344 .OriginBase: 0x0700000000000, // OriginBase
345};
346
347// i386 FreeBSD
348static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
349 .AndMask: 0x000180000000, // AndMask
350 .XorMask: 0x000040000000, // XorMask
351 .ShadowBase: 0x000020000000, // ShadowBase
352 .OriginBase: 0x000700000000, // OriginBase
353};
354
355// x86_64 FreeBSD
356static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
357 .AndMask: 0xc00000000000, // AndMask
358 .XorMask: 0x200000000000, // XorMask
359 .ShadowBase: 0x100000000000, // ShadowBase
360 .OriginBase: 0x380000000000, // OriginBase
361};
362
363// x86_64 NetBSD
364static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
365 .AndMask: 0, // AndMask
366 .XorMask: 0x500000000000, // XorMask
367 .ShadowBase: 0, // ShadowBase
368 .OriginBase: 0x100000000000, // OriginBase
369};
370
371static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
372 .bits32: &Linux_I386_MemoryMapParams,
373 .bits64: &Linux_X86_64_MemoryMapParams,
374};
375
376static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
377 .bits32: nullptr,
378 .bits64: &Linux_MIPS64_MemoryMapParams,
379};
380
381static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
382 .bits32: nullptr,
383 .bits64: &Linux_PowerPC64_MemoryMapParams,
384};
385
386static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
387 .bits32: nullptr,
388 .bits64: &Linux_S390X_MemoryMapParams,
389};
390
391static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
392 .bits32: nullptr,
393 .bits64: &Linux_AArch64_MemoryMapParams,
394};
395
396static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams = {
397 .bits32: nullptr,
398 .bits64: &Linux_LoongArch64_MemoryMapParams,
399};
400
401static const PlatformMemoryMapParams Linux_Hexagon_MemoryMapParams_P = {
402 .bits32: &Linux_Hexagon_MemoryMapParams,
403 .bits64: nullptr,
404};
405
406static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
407 .bits32: nullptr,
408 .bits64: &FreeBSD_AArch64_MemoryMapParams,
409};
410
411static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
412 .bits32: &FreeBSD_I386_MemoryMapParams,
413 .bits64: &FreeBSD_X86_64_MemoryMapParams,
414};
415
416static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
417 .bits32: nullptr,
418 .bits64: &NetBSD_X86_64_MemoryMapParams,
419};
420
421enum OddOrEvenLanes { kBothLanes, kEvenLanes, kOddLanes };
422
423namespace {
424
425/// Instrument functions of a module to detect uninitialized reads.
426///
427/// Instantiating MemorySanitizer inserts the msan runtime library API function
428/// declarations into the module if they don't exist already. Instantiating
429/// ensures the __msan_init function is in the list of global constructors for
430/// the module.
431class MemorySanitizer {
432public:
433 MemorySanitizer(const InstrumentationOptions &Opts, Module &M,
434 MemorySanitizerOptions Options)
435 : Opts(Opts), CompileKernel(Options.Kernel),
436 TrackOrigins(Options.TrackOrigins), Recover(Options.Recover),
437 EagerChecks(Options.EagerChecks) {
438 initializeModule(M);
439 }
440
441 // MSan cannot be moved or copied because of MapParams.
442 MemorySanitizer(MemorySanitizer &&) = delete;
443 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
444 MemorySanitizer(const MemorySanitizer &) = delete;
445 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
446
447 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
448
449private:
450 friend struct MemorySanitizerVisitor;
451 friend struct VarArgHelperBase;
452 friend struct VarArgAMD64Helper;
453 friend struct VarArgAArch64Helper;
454 friend struct VarArgPowerPC64Helper;
455 friend struct VarArgPowerPC32Helper;
456 friend struct VarArgSystemZHelper;
457 friend struct VarArgI386Helper;
458 friend struct VarArgGenericHelper;
459
460 void initializeModule(Module &M);
461 void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
462 void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
463 void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
464
465 template <typename... ArgsTy>
466 FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
467 ArgsTy... Args);
468
469 const InstrumentationOptions &Opts;
470 /// True if we're compiling the Linux kernel.
471 bool CompileKernel;
472 /// Track origins (allocation points) of uninitialized values.
473 int TrackOrigins;
474 bool Recover;
475 bool EagerChecks;
476
477 Triple TargetTriple;
478 LLVMContext *C;
479 Type *IntptrTy; ///< Integer type with the size of a ptr in default AS.
480 Type *OriginTy;
481 PointerType *PtrTy; ///< Integer type with the size of a ptr in default AS.
482
483 // XxxTLS variables represent the per-thread state in MSan and per-task state
484 // in KMSAN.
485 // For the userspace these point to thread-local globals. In the kernel land
486 // they point to the members of a per-task struct obtained via a call to
487 // __msan_get_context_state().
488
489 /// Thread-local shadow storage for function parameters.
490 Value *ParamTLS;
491
492 /// Thread-local origin storage for function parameters.
493 Value *ParamOriginTLS;
494
495 /// Thread-local shadow storage for function return value.
496 Value *RetvalTLS;
497
498 /// Thread-local origin storage for function return value.
499 Value *RetvalOriginTLS;
500
501 /// Thread-local shadow storage for in-register va_arg function.
502 Value *VAArgTLS;
503
504 /// Thread-local shadow storage for in-register va_arg function.
505 Value *VAArgOriginTLS;
506
507 /// Thread-local shadow storage for va_arg overflow area.
508 Value *VAArgOverflowSizeTLS;
509
510 /// Are the instrumentation callbacks set up?
511 bool CallbacksInitialized = false;
512
513 /// The run-time callback to print a warning.
514 FunctionCallee WarningFn;
515
516 // These arrays are indexed by log2(AccessSize).
517 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
518 FunctionCallee MaybeWarningVarSizeFn;
519 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
520
521 /// Run-time helper that generates a new origin value for a stack
522 /// allocation.
523 FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
524 // No description version
525 FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
526
527 /// Run-time helper that poisons stack on function entry.
528 FunctionCallee MsanPoisonStackFn;
529
530 /// Run-time helper that records a store (or any event) of an
531 /// uninitialized value and returns an updated origin id encoding this info.
532 FunctionCallee MsanChainOriginFn;
533
534 /// Run-time helper that paints an origin over a region.
535 FunctionCallee MsanSetOriginFn;
536
537 /// MSan runtime replacements for memmove, memcpy and memset.
538 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
539
540 /// KMSAN callback for task-local function argument shadow.
541 StructType *MsanContextStateTy;
542 FunctionCallee MsanGetContextStateFn;
543
544 /// Functions for poisoning/unpoisoning local variables
545 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
546
547 /// Pair of shadow/origin pointers.
548 Type *MsanMetadata;
549
550 /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
551 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
552 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
553 FunctionCallee MsanMetadataPtrForStore_1_8[4];
554 FunctionCallee MsanInstrumentAsmStoreFn;
555
556 /// Storage for return values of the MsanMetadataPtrXxx functions.
557 Value *MsanMetadataAlloca;
558
559 /// Helper to choose between different MsanMetadataPtrXxx().
560 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
561
562 /// Memory map parameters used in application-to-shadow calculation.
563 const MemoryMapParams *MapParams;
564
565 /// Custom memory map parameters used when -msan-shadow-base or
566 // -msan-origin-base is provided.
567 MemoryMapParams CustomMapParams;
568
569 MDNode *ColdCallWeights;
570
571 /// Branch weights for origin store.
572 MDNode *OriginStoreWeights;
573};
574
575void insertModuleCtor(const InstrumentationOptions &Opts, Module &M) {
576 getOrCreateSanitizerCtorAndInitFunctions(
577 M, CtorName: kMsanModuleCtorName, InitName: kMsanInitName,
578 /*InitArgTypes=*/{},
579 /*InitArgs=*/{},
580 // This callback is invoked when the functions are created the first
581 // time. Hook them into the global ctors list in that case:
582 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
583 if (!Opts.msan_with_comdat) {
584 appendToGlobalCtors(M, F: Ctor, Priority: 0);
585 return;
586 }
587 Comdat *MsanCtorComdat = M.getOrInsertComdat(Name: kMsanModuleCtorName);
588 Ctor->setComdat(MsanCtorComdat);
589 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
590 });
591}
592
593} // end anonymous namespace
594
595MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K, bool EC)
596 : Kernel(K), TrackOrigins(TO), Recover(R), EagerChecks(EC) {}
597
598static MemorySanitizerOptions resolveOptions(const InstrumentationOptions &Opts,
599 MemorySanitizerOptions O) {
600 O.Kernel = valueOr(X: Opts.msan_kernel, Default: O.Kernel);
601 O.TrackOrigins =
602 Opts.msan_track_origins.value_or(u: O.Kernel ? 2 : O.TrackOrigins);
603 O.Recover = valueOr(X: Opts.msan_keep_going, Default: O.Kernel || O.Recover);
604 O.EagerChecks = valueOr(X: Opts.msan_eager_checks, Default: O.EagerChecks);
605 return O;
606}
607
608PreservedAnalyses MemorySanitizerPass::run(Module &M,
609 ModuleAnalysisManager &AM) {
610 // Return early if nosanitize_memory module flag is present for the module.
611 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_memory"))
612 return PreservedAnalyses::all();
613 const InstrumentationOptions &Opts =
614 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
615 MemorySanitizerOptions MSanOpts = resolveOptions(Opts, O: Options);
616 bool Modified = false;
617 if (!MSanOpts.Kernel) {
618 insertModuleCtor(Opts, M);
619 Modified = true;
620 }
621
622 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
623 for (Function &F : M) {
624 if (F.empty())
625 continue;
626 MemorySanitizer Msan(Opts, *F.getParent(), MSanOpts);
627 Modified |=
628 Msan.sanitizeFunction(F, TLI&: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
629 }
630
631 if (!Modified)
632 return PreservedAnalyses::all();
633
634 PreservedAnalyses PA = PreservedAnalyses::none();
635 // GlobalsAA is considered stateless and does not get invalidated unless
636 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
637 // make changes that require GlobalsAA to be invalidated.
638 PA.abandon<GlobalsAA>();
639 return PA;
640}
641
642void MemorySanitizerPass::printPipeline(
643 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
644 static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
645 OS, MapClassName2PassName);
646 OS << '<';
647 if (Options.Recover)
648 OS << "recover;";
649 if (Options.Kernel)
650 OS << "kernel;";
651 if (Options.EagerChecks)
652 OS << "eager-checks;";
653 OS << "track-origins=" << Options.TrackOrigins;
654 OS << '>';
655}
656
657/// Create a non-const global initialized with the given string.
658///
659/// Creates a writable global for Str so that we can pass it to the
660/// run-time lib. Runtime uses first 4 bytes of the string to store the
661/// frame ID, so the string needs to be mutable.
662static GlobalVariable *createPrivateConstGlobalForString(Module &M,
663 StringRef Str) {
664 Constant *StrConst = ConstantDataArray::getString(Context&: M.getContext(), Initializer: Str);
665 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
666 GlobalValue::PrivateLinkage, StrConst, "");
667}
668
669template <typename... ArgsTy>
670FunctionCallee
671MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
672 ArgsTy... Args) {
673 if (TargetTriple.getArch() == Triple::systemz) {
674 // SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
675 return M.getOrInsertFunction(Name, Type::getVoidTy(C&: *C), PtrTy,
676 std::forward<ArgsTy>(Args)...);
677 }
678
679 return M.getOrInsertFunction(Name, MsanMetadata,
680 std::forward<ArgsTy>(Args)...);
681}
682
683/// Create KMSAN API callbacks.
684void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
685 IRBuilder<> IRB(M);
686
687 // These will be initialized in insertKmsanPrologue().
688 RetvalTLS = nullptr;
689 RetvalOriginTLS = nullptr;
690 ParamTLS = nullptr;
691 ParamOriginTLS = nullptr;
692 VAArgTLS = nullptr;
693 VAArgOriginTLS = nullptr;
694 VAArgOverflowSizeTLS = nullptr;
695
696 WarningFn = M.getOrInsertFunction(Name: "__msan_warning",
697 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
698 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
699
700 // Requests the per-task context state (kmsan_context_state*) from the
701 // runtime library.
702 MsanContextStateTy = StructType::get(
703 elt1: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
704 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8),
705 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
706 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8), /* va_arg_origin */
707 elts: IRB.getInt64Ty(), elts: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4), elts: OriginTy,
708 elts: OriginTy);
709 MsanGetContextStateFn =
710 M.getOrInsertFunction(Name: "__msan_get_context_state", RetTy: PtrTy);
711
712 MsanMetadata = StructType::get(elt1: PtrTy, elts: PtrTy);
713
714 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
715 std::string name_load =
716 "__msan_metadata_ptr_for_load_" + std::to_string(val: size);
717 std::string name_store =
718 "__msan_metadata_ptr_for_store_" + std::to_string(val: size);
719 MsanMetadataPtrForLoad_1_8[ind] =
720 getOrInsertMsanMetadataFunction(M, Name: name_load, Args: PtrTy);
721 MsanMetadataPtrForStore_1_8[ind] =
722 getOrInsertMsanMetadataFunction(M, Name: name_store, Args: PtrTy);
723 }
724
725 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
726 M, Name: "__msan_metadata_ptr_for_load_n", Args: PtrTy, Args: IntptrTy);
727 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
728 M, Name: "__msan_metadata_ptr_for_store_n", Args: PtrTy, Args: IntptrTy);
729
730 // Functions for poisoning and unpoisoning memory.
731 MsanPoisonAllocaFn = M.getOrInsertFunction(
732 Name: "__msan_poison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
733 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
734 Name: "__msan_unpoison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
735}
736
737static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
738 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&] {
739 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
740 nullptr, Name, nullptr,
741 GlobalVariable::InitialExecTLSModel);
742 });
743}
744
745/// Insert declarations for userspace-specific functions and globals.
746void MemorySanitizer::createUserspaceApi(Module &M,
747 const TargetLibraryInfo &TLI) {
748 IRBuilder<> IRB(M);
749
750 // Create the callback.
751 // FIXME: this function should have "Cold" calling conv,
752 // which is not yet implemented.
753 if (TrackOrigins) {
754 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
755 : "__msan_warning_with_origin_noreturn";
756 WarningFn = M.getOrInsertFunction(Name: WarningFnName,
757 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
758 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
759 } else {
760 StringRef WarningFnName =
761 Recover ? "__msan_warning" : "__msan_warning_noreturn";
762 WarningFn = M.getOrInsertFunction(Name: WarningFnName, RetTy: IRB.getVoidTy());
763 }
764
765 // Create the global TLS variables.
766 RetvalTLS =
767 getOrInsertGlobal(M, Name: "__msan_retval_tls",
768 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8));
769
770 RetvalOriginTLS = getOrInsertGlobal(M, Name: "__msan_retval_origin_tls", Ty: OriginTy);
771
772 ParamTLS =
773 getOrInsertGlobal(M, Name: "__msan_param_tls",
774 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
775
776 ParamOriginTLS =
777 getOrInsertGlobal(M, Name: "__msan_param_origin_tls",
778 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
779
780 VAArgTLS =
781 getOrInsertGlobal(M, Name: "__msan_va_arg_tls",
782 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
783
784 VAArgOriginTLS =
785 getOrInsertGlobal(M, Name: "__msan_va_arg_origin_tls",
786 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
787
788 VAArgOverflowSizeTLS = getOrInsertGlobal(M, Name: "__msan_va_arg_overflow_size_tls",
789 Ty: IRB.getIntPtrTy(DL: M.getDataLayout()));
790
791 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
792 AccessSizeIndex++) {
793 unsigned AccessSize = 1 << AccessSizeIndex;
794 std::string FunctionName = "__msan_maybe_warning_" + itostr(X: AccessSize);
795 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
796 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 1}, /*Signed=*/false),
797 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: IRB.getInt32Ty());
798 MaybeWarningVarSizeFn = M.getOrInsertFunction(
799 Name: "__msan_maybe_warning_N", AttributeList: TLI.getAttrList(C, ArgNos: {}, /*Signed=*/false),
800 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IRB.getInt64Ty(), Args: IRB.getInt32Ty());
801 FunctionName = "__msan_maybe_store_origin_" + itostr(X: AccessSize);
802 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
803 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 2}, /*Signed=*/false),
804 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: PtrTy,
805 Args: IRB.getInt32Ty());
806 }
807
808 MsanSetAllocaOriginWithDescriptionFn =
809 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_with_descr",
810 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy, Args: PtrTy);
811 MsanSetAllocaOriginNoDescriptionFn =
812 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_no_descr",
813 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
814 MsanPoisonStackFn = M.getOrInsertFunction(Name: "__msan_poison_stack",
815 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
816}
817
818/// Insert extern declaration of runtime-provided functions and globals.
819void MemorySanitizer::initializeCallbacks(Module &M,
820 const TargetLibraryInfo &TLI) {
821 // Only do this once.
822 if (CallbacksInitialized)
823 return;
824
825 IRBuilder<> IRB(M);
826 // Initialize callbacks that are common for kernel and userspace
827 // instrumentation.
828 MsanChainOriginFn = M.getOrInsertFunction(
829 Name: "__msan_chain_origin",
830 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false, /*Ret=*/true), RetTy: IRB.getInt32Ty(),
831 Args: IRB.getInt32Ty());
832 MsanSetOriginFn = M.getOrInsertFunction(
833 Name: "__msan_set_origin", AttributeList: TLI.getAttrList(C, ArgNos: {2}, /*Signed=*/false),
834 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: IRB.getInt32Ty());
835 MemmoveFn =
836 M.getOrInsertFunction(Name: "__msan_memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
837 MemcpyFn =
838 M.getOrInsertFunction(Name: "__msan_memcpy", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
839 MemsetFn = M.getOrInsertFunction(Name: "__msan_memset",
840 AttributeList: TLI.getAttrList(C, ArgNos: {1}, /*Signed=*/true),
841 RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
842
843 MsanInstrumentAsmStoreFn = M.getOrInsertFunction(
844 Name: "__msan_instrument_asm_store", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
845
846 if (CompileKernel) {
847 createKernelApi(M, TLI);
848 } else {
849 createUserspaceApi(M, TLI);
850 }
851 CallbacksInitialized = true;
852}
853
854FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
855 int size) {
856 FunctionCallee *Fns =
857 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
858 switch (size) {
859 case 1:
860 return Fns[0];
861 case 2:
862 return Fns[1];
863 case 4:
864 return Fns[2];
865 case 8:
866 return Fns[3];
867 default:
868 return nullptr;
869 }
870}
871
872/// Module-level initialization.
873///
874/// inserts a call to __msan_init to the module's constructor list.
875void MemorySanitizer::initializeModule(Module &M) {
876 auto &DL = M.getDataLayout();
877
878 TargetTriple = M.getTargetTriple();
879
880 bool ShadowPassed = Opts.msan_shadow_base.has_value();
881 bool OriginPassed = Opts.msan_origin_base.has_value();
882 // Check the overrides first
883 if (ShadowPassed || OriginPassed) {
884 CustomMapParams.AndMask = Opts.msan_and_mask;
885 CustomMapParams.XorMask = Opts.msan_xor_mask;
886 CustomMapParams.ShadowBase = Opts.msan_shadow_base.value_or(u: 0);
887 CustomMapParams.OriginBase = Opts.msan_origin_base.value_or(u: 0);
888 MapParams = &CustomMapParams;
889 } else {
890 switch (TargetTriple.getOS()) {
891 case Triple::FreeBSD:
892 switch (TargetTriple.getArch()) {
893 case Triple::aarch64:
894 MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
895 break;
896 case Triple::x86_64:
897 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
898 break;
899 case Triple::x86:
900 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
901 break;
902 default:
903 report_fatal_error(reason: "unsupported architecture");
904 }
905 break;
906 case Triple::NetBSD:
907 switch (TargetTriple.getArch()) {
908 case Triple::x86_64:
909 MapParams = NetBSD_X86_MemoryMapParams.bits64;
910 break;
911 default:
912 report_fatal_error(reason: "unsupported architecture");
913 }
914 break;
915 case Triple::Linux:
916 switch (TargetTriple.getArch()) {
917 case Triple::x86_64:
918 MapParams = Linux_X86_MemoryMapParams.bits64;
919 break;
920 case Triple::x86:
921 MapParams = Linux_X86_MemoryMapParams.bits32;
922 break;
923 case Triple::mips64:
924 case Triple::mips64el:
925 MapParams = Linux_MIPS_MemoryMapParams.bits64;
926 break;
927 case Triple::ppc64:
928 case Triple::ppc64le:
929 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
930 break;
931 case Triple::systemz:
932 MapParams = Linux_S390_MemoryMapParams.bits64;
933 break;
934 case Triple::aarch64:
935 case Triple::aarch64_be:
936 MapParams = Linux_ARM_MemoryMapParams.bits64;
937 break;
938 case Triple::loongarch64:
939 MapParams = Linux_LoongArch_MemoryMapParams.bits64;
940 break;
941 case Triple::hexagon:
942 MapParams = Linux_Hexagon_MemoryMapParams_P.bits32;
943 break;
944 default:
945 report_fatal_error(reason: "unsupported architecture");
946 }
947 break;
948 default:
949 report_fatal_error(reason: "unsupported operating system");
950 }
951 }
952
953 C = &(M.getContext());
954 IRBuilder<> IRB(M);
955 IntptrTy = IRB.getIntPtrTy(DL);
956 OriginTy = IRB.getInt32Ty();
957 PtrTy = IRB.getPtrTy();
958
959 ColdCallWeights = MDBuilder(*C).createUnlikelyBranchWeights();
960 OriginStoreWeights = MDBuilder(*C).createUnlikelyBranchWeights();
961
962 if (!CompileKernel) {
963 if (TrackOrigins)
964 M.getOrInsertGlobal(Name: "__msan_track_origins", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
965 return new GlobalVariable(
966 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
967 IRB.getInt32(C: TrackOrigins), "__msan_track_origins");
968 });
969
970 if (Recover)
971 M.getOrInsertGlobal(Name: "__msan_keep_going", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
972 return new GlobalVariable(M, IRB.getInt32Ty(), true,
973 GlobalValue::WeakODRLinkage,
974 IRB.getInt32(C: Recover), "__msan_keep_going");
975 });
976 }
977}
978
979namespace {
980
981/// A helper class that handles instrumentation of VarArg
982/// functions on a particular platform.
983///
984/// Implementations are expected to insert the instrumentation
985/// necessary to propagate argument shadow through VarArg function
986/// calls. Visit* methods are called during an InstVisitor pass over
987/// the function, and should avoid creating new basic blocks. A new
988/// instance of this class is created for each instrumented function.
989struct VarArgHelper {
990 virtual ~VarArgHelper() = default;
991
992 /// Visit a CallBase.
993 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
994
995 /// Visit a va_start call.
996 virtual void visitVAStartInst(VAStartInst &I) = 0;
997
998 /// Visit a va_copy call.
999 virtual void visitVACopyInst(VACopyInst &I) = 0;
1000
1001 /// Finalize function instrumentation.
1002 ///
1003 /// This method is called after visiting all interesting (see above)
1004 /// instructions in a function.
1005 virtual void finalizeInstrumentation() = 0;
1006};
1007
1008struct MemorySanitizerVisitor;
1009
1010} // end anonymous namespace
1011
1012static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1013 MemorySanitizerVisitor &Visitor);
1014
1015static unsigned TypeSizeToSizeIndex(TypeSize TS) {
1016 if (TS.isScalable())
1017 // Scalable types unconditionally take slowpaths.
1018 return kNumberOfAccessSizes;
1019 unsigned TypeSizeFixed = TS.getFixedValue();
1020 if (TypeSizeFixed <= 8)
1021 return 0;
1022 return Log2_32_Ceil(Value: (TypeSizeFixed + 7) / 8);
1023}
1024
1025namespace {
1026
1027/// Helper class to attach debug information of the given instruction onto new
1028/// instructions inserted after.
1029class NextNodeIRBuilder : public IRBuilder<> {
1030public:
1031 explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1032 SetCurrentDebugLocation(IP->getDebugLoc());
1033 }
1034};
1035
1036/// This class does all the work for a given function. Store and Load
1037/// instructions store and load corresponding shadow and origin
1038/// values. Most instructions propagate shadow from arguments to their
1039/// return values. Certain instructions (most importantly, BranchInst)
1040/// test their argument shadow and print reports (with a runtime call) if it's
1041/// non-zero.
1042struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1043 const InstrumentationOptions &Opts;
1044 Function &F;
1045 MemorySanitizer &MS;
1046 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1047 ValueMap<Value *, Value *> ShadowMap, OriginMap;
1048 std::unique_ptr<VarArgHelper> VAHelper;
1049 const TargetLibraryInfo *TLI;
1050 Instruction *FnPrologueEnd;
1051 SmallVector<Instruction *, 16> Instructions;
1052
1053 // The following flags disable parts of MSan instrumentation based on
1054 // exclusion list contents and command-line options.
1055 bool InsertChecks;
1056 bool PropagateShadow;
1057 bool PoisonStack;
1058 bool PoisonUndef;
1059 bool PoisonUndefVectors;
1060
1061 struct ShadowOriginAndInsertPoint {
1062 Value *Shadow;
1063 Value *Origin;
1064 Instruction *OrigIns;
1065
1066 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1067 : Shadow(S), Origin(O), OrigIns(I) {}
1068 };
1069 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1070 DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
1071 SmallSetVector<AllocaInst *, 16> AllocaSet;
1072 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1073 SmallVector<StoreInst *, 16> StoreList;
1074 int64_t SplittableBlocksCount = 0;
1075
1076 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1077 const TargetLibraryInfo &TLI)
1078 : Opts(MS.Opts), F(F), MS(MS), VAHelper(CreateVarArgHelper(Func&: F, Msan&: MS, Visitor&: *this)),
1079 TLI(&TLI) {
1080 bool SanitizeFunction = F.hasFnAttribute(Kind: Attribute::SanitizeMemory) &&
1081 !Opts.msan_disable_checks;
1082 InsertChecks = SanitizeFunction;
1083 PropagateShadow = SanitizeFunction;
1084 PoisonStack = SanitizeFunction && Opts.msan_poison_stack;
1085 PoisonUndef = SanitizeFunction && Opts.msan_poison_undef;
1086 PoisonUndefVectors = SanitizeFunction && Opts.msan_poison_undef_vectors;
1087
1088 // In the presence of unreachable blocks, we may see Phi nodes with
1089 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1090 // blocks, such nodes will not have any shadow value associated with them.
1091 // It's easier to remove unreachable blocks than deal with missing shadow.
1092 removeUnreachableBlocks(F);
1093
1094 MS.initializeCallbacks(M&: *F.getParent(), TLI);
1095 FnPrologueEnd =
1096 IRBuilder<>(F.getEntryBlock().getFirstNonPHIIt())
1097 .CreateIntrinsicWithoutFolding(ID: Intrinsic::donothing, Args: {});
1098
1099 if (MS.CompileKernel) {
1100 IRBuilder<> IRB(FnPrologueEnd);
1101 insertKmsanPrologue(IRB);
1102 }
1103
1104 LLVM_DEBUG(if (!InsertChecks) dbgs()
1105 << "MemorySanitizer is not inserting checks into '"
1106 << F.getName() << "'\n");
1107 }
1108
1109 bool instrumentWithCalls(Value *V) {
1110 // Constants likely will be eliminated by follow-up passes.
1111 if (isa<Constant>(Val: V))
1112 return false;
1113 ++SplittableBlocksCount;
1114 return Opts.msan_instrumentation_with_call_threshold >= 0 &&
1115 SplittableBlocksCount >
1116 Opts.msan_instrumentation_with_call_threshold;
1117 }
1118
1119 bool isInPrologue(Instruction &I) {
1120 return I.getParent() == FnPrologueEnd->getParent() &&
1121 (&I == FnPrologueEnd || I.comesBefore(Other: FnPrologueEnd));
1122 }
1123
1124 // Creates a new origin and records the stack trace. In general we can call
1125 // this function for any origin manipulation we like. However it will cost
1126 // runtime resources. So use this wisely only if it can provide additional
1127 // information helpful to a user.
1128 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1129 if (MS.TrackOrigins <= 1)
1130 return V;
1131 return IRB.CreateCall(Callee: MS.MsanChainOriginFn, Args: V);
1132 }
1133
1134 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1135 const DataLayout &DL = F.getDataLayout();
1136 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1137 if (IntptrSize == kOriginSize)
1138 return Origin;
1139 assert(IntptrSize == kOriginSize * 2);
1140 Origin = IRB.CreateIntCast(V: Origin, DestTy: MS.IntptrTy, /* isSigned */ false);
1141 return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: kOriginSize * 8));
1142 }
1143
1144 /// Fill memory range with the given origin value.
1145 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1146 TypeSize TS, Align Alignment) {
1147 const DataLayout &DL = F.getDataLayout();
1148 const Align IntptrAlignment = DL.getABITypeAlign(Ty: MS.IntptrTy);
1149 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1150 assert(IntptrAlignment >= kMinOriginAlignment);
1151 assert(IntptrSize >= kOriginSize);
1152
1153 // Note: The loop based formation works for fixed length vectors too,
1154 // however we prefer to unroll and specialize alignment below.
1155 if (TS.isScalable()) {
1156 Value *Size = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size: TS);
1157 Value *RoundUp =
1158 IRB.CreateAdd(LHS: Size, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize - 1));
1159 Value *End =
1160 IRB.CreateUDiv(LHS: RoundUp, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize));
1161 auto [InsertPt, Index] =
1162 SplitBlockAndInsertSimpleForLoop(End, SplitBefore: IRB.GetInsertPoint());
1163 IRB.SetInsertPoint(InsertPt);
1164
1165 Value *GEP = IRB.CreateGEP(Ty: MS.OriginTy, Ptr: OriginPtr, IdxList: Index);
1166 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: kMinOriginAlignment);
1167 return;
1168 }
1169
1170 unsigned Size = TS.getFixedValue();
1171
1172 unsigned Ofs = 0;
1173 Align CurrentAlignment = Alignment;
1174 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1175 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1176 Value *IntptrOriginPtr = IRB.CreatePointerCast(V: OriginPtr, DestTy: MS.PtrTy);
1177 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1178 Value *Ptr = i ? IRB.CreateConstGEP1_32(Ty: MS.IntptrTy, Ptr: IntptrOriginPtr, Idx0: i)
1179 : IntptrOriginPtr;
1180 IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
1181 Ofs += IntptrSize / kOriginSize;
1182 CurrentAlignment = IntptrAlignment;
1183 }
1184 }
1185
1186 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1187 Value *GEP =
1188 i ? IRB.CreateConstGEP1_32(Ty: MS.OriginTy, Ptr: OriginPtr, Idx0: i) : OriginPtr;
1189 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
1190 CurrentAlignment = kMinOriginAlignment;
1191 }
1192 }
1193
1194 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1195 Value *OriginPtr, Align Alignment) {
1196 const DataLayout &DL = F.getDataLayout();
1197 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1198 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
1199 // ZExt cannot convert between vector and scalar
1200 Value *ConvertedShadow = convertShadowToScalar(V: Shadow, IRB);
1201 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1202 if (!Opts.msan_check_constant_shadow || ConstantShadow->isNullValue()) {
1203 // Origin is not needed: value is initialized or const shadow is
1204 // ignored.
1205 return;
1206 }
1207 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1208 // Copy origin as the value is definitely uninitialized.
1209 paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), OriginPtr, TS: StoreSize,
1210 Alignment: OriginAlignment);
1211 return;
1212 }
1213 // Fallback to runtime check, which still can be optimized out later.
1214 }
1215
1216 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1217 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1218 if (instrumentWithCalls(V: ConvertedShadow) &&
1219 SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1220 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1221 Value *ConvertedShadow2 =
1222 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1223 CallBase *CB = IRB.CreateCall(Callee: Fn, Args: {ConvertedShadow2, Addr, Origin});
1224 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1225 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1226 } else {
1227 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1228 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1229 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: MS.OriginStoreWeights);
1230 IRBuilder<> IRBNew(CheckTerm);
1231 paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), OriginPtr, TS: StoreSize,
1232 Alignment: OriginAlignment);
1233 }
1234 }
1235
1236 void materializeStores() {
1237 for (StoreInst *SI : StoreList) {
1238 IRBuilder<> IRB(SI);
1239 Value *Val = SI->getValueOperand();
1240 Value *Addr = SI->getPointerOperand();
1241 Value *Shadow = SI->isAtomic() ? getCleanShadow(V: Val) : getShadow(V: Val);
1242 Value *ShadowPtr, *OriginPtr;
1243 Type *ShadowTy = Shadow->getType();
1244 const Align Alignment = SI->getAlign();
1245 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1246 std::tie(args&: ShadowPtr, args&: OriginPtr) =
1247 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1248
1249 [[maybe_unused]] StoreInst *NewSI =
1250 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Alignment);
1251 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1252
1253 if (SI->isAtomic())
1254 SI->setOrdering(addReleaseOrdering(a: SI->getOrdering()));
1255
1256 if (MS.TrackOrigins && !SI->isAtomic())
1257 storeOrigin(IRB, Addr, Shadow, Origin: getOrigin(V: Val), OriginPtr,
1258 Alignment: OriginAlignment);
1259 }
1260 }
1261
1262 // Returns true if Debug Location corresponds to multiple warnings.
1263 bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1264 if (MS.TrackOrigins < 2)
1265 return false;
1266
1267 if (LazyWarningDebugLocationCount.empty())
1268 for (const auto &I : InstrumentationList)
1269 ++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1270
1271 return LazyWarningDebugLocationCount[DebugLoc] >=
1272 Opts.msan_disambiguate_warning_threshold;
1273 }
1274
1275 /// Helper function to insert a warning at IRB's current insert point.
1276 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1277 if (!Origin)
1278 Origin = (Value *)IRB.getInt32(C: 0);
1279 assert(Origin->getType()->isIntegerTy());
1280
1281 if (shouldDisambiguateWarningLocation(DebugLoc: IRB.getCurrentDebugLocation())) {
1282 // Try to create additional origin with debug info of the last origin
1283 // instruction. It may provide additional information to the user.
1284 if (Instruction *OI = dyn_cast_or_null<Instruction>(Val: Origin)) {
1285 assert(MS.TrackOrigins);
1286 auto NewDebugLoc = OI->getDebugLoc();
1287 // Origin update with missing or the same debug location provides no
1288 // additional value.
1289 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1290 // Insert update just before the check, so we call runtime only just
1291 // before the report.
1292 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1293 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1294 Origin = updateOrigin(V: Origin, IRB&: IRBOrigin);
1295 }
1296 }
1297 }
1298
1299 if (MS.CompileKernel || MS.TrackOrigins)
1300 IRB.CreateCall(Callee: MS.WarningFn, Args: Origin)->setCannotMerge();
1301 else
1302 IRB.CreateCall(Callee: MS.WarningFn)->setCannotMerge();
1303 // FIXME: Insert UnreachableInst if !MS.Recover?
1304 // This may invalidate some of the following checks and needs to be done
1305 // at the very end.
1306 }
1307
1308 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1309 Value *Origin) {
1310 const DataLayout &DL = F.getDataLayout();
1311 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1312 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1313 if (instrumentWithCalls(V: ConvertedShadow) && !MS.CompileKernel) {
1314 // ZExt cannot convert between vector and scalar
1315 ConvertedShadow = convertShadowToScalar(V: ConvertedShadow, IRB);
1316 Value *ConvertedShadow2 =
1317 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1318
1319 if (SizeIndex < kNumberOfAccessSizes) {
1320 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1321 CallBase *CB = IRB.CreateCall(
1322 Callee: Fn,
1323 Args: {ConvertedShadow2,
1324 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1325 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1326 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1327 } else {
1328 FunctionCallee Fn = MS.MaybeWarningVarSizeFn;
1329 Value *ShadowAlloca = IRB.CreateAlloca(Ty: ConvertedShadow2->getType(), AddrSpace: 0u);
1330 IRB.CreateStore(Val: ConvertedShadow2, Ptr: ShadowAlloca);
1331 unsigned ShadowSize = DL.getTypeAllocSize(Ty: ConvertedShadow2->getType());
1332 CallBase *CB = IRB.CreateCall(
1333 Callee: Fn,
1334 Args: {ShadowAlloca, ConstantInt::get(Ty: IRB.getInt64Ty(), V: ShadowSize),
1335 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1336 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1337 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1338 }
1339 } else {
1340 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1341 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1342 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(),
1343 /* Unreachable */ !MS.Recover, BranchWeights: MS.ColdCallWeights);
1344
1345 IRB.SetInsertPoint(CheckTerm);
1346 insertWarningFn(IRB, Origin);
1347 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1348 }
1349 }
1350
1351 void materializeInstructionChecks(
1352 ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1353 const DataLayout &DL = F.getDataLayout();
1354 // Disable combining in some cases. TrackOrigins checks each shadow to pick
1355 // correct origin.
1356 bool Combine = !MS.TrackOrigins;
1357 Instruction *Instruction = InstructionChecks.front().OrigIns;
1358 Value *Shadow = nullptr;
1359 for (const auto &ShadowData : InstructionChecks) {
1360 assert(ShadowData.OrigIns == Instruction);
1361 IRBuilder<> IRB(Instruction);
1362
1363 Value *ConvertedShadow = ShadowData.Shadow;
1364
1365 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1366 if (!Opts.msan_check_constant_shadow || ConstantShadow->isNullValue()) {
1367 // Skip, value is initialized or const shadow is ignored.
1368 continue;
1369 }
1370 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1371 // Report as the value is definitely uninitialized.
1372 insertWarningFn(IRB, Origin: ShadowData.Origin);
1373 if (!MS.Recover)
1374 return; // Always fail and stop here, not need to check the rest.
1375 // Skip entire instruction,
1376 continue;
1377 }
1378 // Fallback to runtime check, which still can be optimized out later.
1379 }
1380
1381 if (!Combine) {
1382 materializeOneCheck(IRB, ConvertedShadow, Origin: ShadowData.Origin);
1383 continue;
1384 }
1385
1386 if (!Shadow) {
1387 Shadow = ConvertedShadow;
1388 continue;
1389 }
1390
1391 Shadow = convertToBool(V: Shadow, IRB, name: "_mscmp");
1392 ConvertedShadow = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1393 Shadow = IRB.CreateOr(LHS: Shadow, RHS: ConvertedShadow, Name: "_msor");
1394 }
1395
1396 if (Shadow) {
1397 assert(Combine);
1398 IRBuilder<> IRB(Instruction);
1399 materializeOneCheck(IRB, ConvertedShadow: Shadow, Origin: nullptr);
1400 }
1401 }
1402
1403 static bool isAArch64SVCount(Type *Ty) {
1404 if (TargetExtType *TTy = dyn_cast<TargetExtType>(Val: Ty))
1405 return TTy->getName() == "aarch64.svcount";
1406 return false;
1407 }
1408
1409 // This is intended to match the "AArch64 Predicate-as-Counter Type" (aka
1410 // 'target("aarch64.svcount")', but not e.g., <vscale x 4 x i32>.
1411 static bool isScalableNonVectorType(Type *Ty) {
1412 if (!isAArch64SVCount(Ty))
1413 LLVM_DEBUG(dbgs() << "isScalableNonVectorType: Unexpected type " << *Ty
1414 << "\n");
1415
1416 return Ty->isScalableTy() && !isa<VectorType>(Val: Ty);
1417 }
1418
1419 void materializeChecks() {
1420#ifndef NDEBUG
1421 // For assert below.
1422 SmallPtrSet<Instruction *, 16> Done;
1423#endif
1424
1425 for (auto I = InstrumentationList.begin();
1426 I != InstrumentationList.end();) {
1427 auto OrigIns = I->OrigIns;
1428 // Checks are grouped by the original instruction. We call all
1429 // `insertShadowCheck` for an instruction at once.
1430 assert(Done.insert(OrigIns).second);
1431 auto J = std::find_if(first: I + 1, last: InstrumentationList.end(),
1432 pred: [OrigIns](const ShadowOriginAndInsertPoint &R) {
1433 return OrigIns != R.OrigIns;
1434 });
1435 // Process all checks of instruction at once.
1436 materializeInstructionChecks(InstructionChecks: ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1437 I = J;
1438 }
1439
1440 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1441 }
1442
1443 // Returns the last instruction in the new prologue
1444 void insertKmsanPrologue(IRBuilder<> &IRB) {
1445 Value *ContextState = IRB.CreateCall(Callee: MS.MsanGetContextStateFn, Args: {});
1446 Constant *Zero = IRB.getInt32(C: 0);
1447 MS.ParamTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1448 IdxList: {Zero, IRB.getInt32(C: 0)}, Name: "param_shadow");
1449 MS.RetvalTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1450 IdxList: {Zero, IRB.getInt32(C: 1)}, Name: "retval_shadow");
1451 MS.VAArgTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1452 IdxList: {Zero, IRB.getInt32(C: 2)}, Name: "va_arg_shadow");
1453 MS.VAArgOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1454 IdxList: {Zero, IRB.getInt32(C: 3)}, Name: "va_arg_origin");
1455 MS.VAArgOverflowSizeTLS =
1456 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1457 IdxList: {Zero, IRB.getInt32(C: 4)}, Name: "va_arg_overflow_size");
1458 MS.ParamOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1459 IdxList: {Zero, IRB.getInt32(C: 5)}, Name: "param_origin");
1460 MS.RetvalOriginTLS =
1461 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1462 IdxList: {Zero, IRB.getInt32(C: 6)}, Name: "retval_origin");
1463 if (MS.TargetTriple.getArch() == Triple::systemz)
1464 MS.MsanMetadataAlloca = IRB.CreateAlloca(Ty: MS.MsanMetadata, AddrSpace: 0u);
1465 }
1466
1467 /// Add MemorySanitizer instrumentation to a function.
1468 bool runOnFunction() {
1469 // Iterate all BBs in depth-first order and create shadow instructions
1470 // for all instructions (where applicable).
1471 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1472 for (BasicBlock *BB : depth_first(G: FnPrologueEnd->getParent()))
1473 visit(BB&: *BB);
1474
1475 // `visit` above only collects instructions. Process them after iterating
1476 // CFG to avoid requirement on CFG transformations.
1477 for (Instruction *I : Instructions)
1478 InstVisitor<MemorySanitizerVisitor>::visit(I&: *I);
1479
1480 // Finalize PHI nodes.
1481 for (PHINode *PN : ShadowPHINodes) {
1482 PHINode *PNS = cast<PHINode>(Val: getShadow(V: PN));
1483 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(Val: getOrigin(V: PN)) : nullptr;
1484 size_t NumValues = PN->getNumIncomingValues();
1485 for (size_t v = 0; v < NumValues; v++) {
1486 PNS->addIncoming(V: getShadow(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1487 if (PNO)
1488 PNO->addIncoming(V: getOrigin(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1489 }
1490 }
1491
1492 VAHelper->finalizeInstrumentation();
1493
1494 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1495 // instrumenting only allocas.
1496 if (Opts.msan_handle_lifetime_intrinsics) {
1497 for (auto Item : LifetimeStartList) {
1498 instrumentAlloca(I&: *Item.second, InsPoint: Item.first);
1499 AllocaSet.remove(X: Item.second);
1500 }
1501 }
1502 // Poison the allocas for which we didn't instrument the corresponding
1503 // lifetime intrinsics.
1504 for (AllocaInst *AI : AllocaSet)
1505 instrumentAlloca(I&: *AI);
1506
1507 // Insert shadow value checks.
1508 materializeChecks();
1509
1510 // Delayed instrumentation of StoreInst.
1511 // This may not add new address checks.
1512 materializeStores();
1513
1514 return true;
1515 }
1516
1517 /// Compute the shadow type that corresponds to a given Value.
1518 Type *getShadowTy(Value *V) { return getShadowTy(OrigTy: V->getType()); }
1519
1520 /// Compute the shadow type that corresponds to a given Type.
1521 Type *getShadowTy(Type *OrigTy) {
1522 if (!OrigTy->isSized()) {
1523 return nullptr;
1524 }
1525 // For integer type, shadow is the same as the original type.
1526 // This may return weird-sized types like i1.
1527 if (IntegerType *IT = dyn_cast<IntegerType>(Val: OrigTy))
1528 return IT;
1529 const DataLayout &DL = F.getDataLayout();
1530 if (VectorType *VT = dyn_cast<VectorType>(Val: OrigTy)) {
1531 uint32_t EltSize = DL.getTypeSizeInBits(Ty: VT->getElementType());
1532 return VectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSize),
1533 EC: VT->getElementCount());
1534 }
1535 if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy)) {
1536 return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1537 NumElements: AT->getNumElements());
1538 }
1539 if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1540 SmallVector<Type *, 4> Elements;
1541 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1542 Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: i)));
1543 StructType *Res = StructType::get(Context&: *MS.C, Elements, isPacked: ST->isPacked());
1544 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1545 return Res;
1546 }
1547 if (isScalableNonVectorType(Ty: OrigTy)) {
1548 LLVM_DEBUG(dbgs() << "getShadowTy: Scalable non-vector type: " << *OrigTy
1549 << "\n");
1550 return OrigTy;
1551 }
1552
1553 uint32_t TypeSize = DL.getTypeSizeInBits(Ty: OrigTy);
1554 return IntegerType::get(C&: *MS.C, NumBits: TypeSize);
1555 }
1556
1557 /// Extract combined shadow of struct elements as a bool
1558 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1559 IRBuilder<> &IRB) {
1560 Value *FalseVal = IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1561 Value *Aggregator = FalseVal;
1562
1563 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1564 // Combine by ORing together each element's bool shadow
1565 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1566 Value *ShadowBool = convertToBool(V: ShadowItem, IRB);
1567
1568 if (Aggregator != FalseVal)
1569 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowBool);
1570 else
1571 Aggregator = ShadowBool;
1572 }
1573
1574 return Aggregator;
1575 }
1576
1577 // Extract combined shadow of array elements
1578 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1579 IRBuilder<> &IRB) {
1580 if (!Array->getNumElements())
1581 return IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1582
1583 Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
1584 Value *Aggregator = convertShadowToScalar(V: FirstItem, IRB);
1585
1586 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1587 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1588 Value *ShadowInner = convertShadowToScalar(V: ShadowItem, IRB);
1589 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
1590 }
1591 return Aggregator;
1592 }
1593
1594 /// Convert a shadow value to it's flattened variant. The resulting
1595 /// shadow may not necessarily have the same bit width as the input
1596 /// value, but it will always be comparable to zero.
1597 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1598 if (StructType *Struct = dyn_cast<StructType>(Val: V->getType()))
1599 return collapseStructShadow(Struct, Shadow: V, IRB);
1600 if (ArrayType *Array = dyn_cast<ArrayType>(Val: V->getType()))
1601 return collapseArrayShadow(Array, Shadow: V, IRB);
1602 if (isa<VectorType>(Val: V->getType())) {
1603 if (isa<ScalableVectorType>(Val: V->getType()))
1604 return convertShadowToScalar(V: IRB.CreateOrReduce(Src: V), IRB);
1605 unsigned BitWidth =
1606 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1607 return IRB.CreateBitCast(V, DestTy: IntegerType::get(C&: *MS.C, NumBits: BitWidth));
1608 }
1609 return V;
1610 }
1611
1612 // Convert a scalar value to an i1 by comparing with 0
1613 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1614 Type *VTy = V->getType();
1615 if (!VTy->isIntegerTy())
1616 return convertToBool(V: convertShadowToScalar(V, IRB), IRB, name);
1617 if (VTy->getIntegerBitWidth() == 1)
1618 // Just converting a bool to a bool, so do nothing.
1619 return V;
1620 return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name: name);
1621 }
1622
1623 Type *ptrToIntPtrType(Type *PtrTy) const {
1624 if (VectorType *VectTy = dyn_cast<VectorType>(Val: PtrTy)) {
1625 return VectorType::get(ElementType: ptrToIntPtrType(PtrTy: VectTy->getElementType()),
1626 EC: VectTy->getElementCount());
1627 }
1628 assert(PtrTy->isIntOrPtrTy());
1629 return MS.IntptrTy;
1630 }
1631
1632 Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1633 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1634 return VectorType::get(
1635 ElementType: getPtrToShadowPtrType(IntPtrTy: VectTy->getElementType(), ShadowTy),
1636 EC: VectTy->getElementCount());
1637 }
1638 assert(IntPtrTy == MS.IntptrTy);
1639 return MS.PtrTy;
1640 }
1641
1642 Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1643 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1644 return ConstantVector::getSplat(
1645 EC: VectTy->getElementCount(),
1646 Elt: constToIntPtr(IntPtrTy: VectTy->getElementType(), C));
1647 }
1648 assert(IntPtrTy == MS.IntptrTy);
1649 // TODO: Avoid implicit trunc?
1650 // See https://github.com/llvm/llvm-project/issues/112510.
1651 return ConstantInt::get(Ty: MS.IntptrTy, V: C, /*IsSigned=*/false,
1652 /*ImplicitTrunc=*/true);
1653 }
1654
1655 /// Returns the integer shadow offset that corresponds to a given
1656 /// application address, whereby:
1657 ///
1658 /// Offset = (Addr & ~AndMask) ^ XorMask
1659 /// Shadow = ShadowBase + Offset
1660 /// Origin = (OriginBase + Offset) & ~Alignment
1661 ///
1662 /// Note: for efficiency, many shadow mappings only require use the XorMask
1663 /// and OriginBase; the AndMask and ShadowBase are often zero.
1664 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1665 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1666 Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1667
1668 if (uint64_t AndMask = MS.MapParams->AndMask)
1669 OffsetLong = IRB.CreateAnd(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~AndMask));
1670
1671 if (uint64_t XorMask = MS.MapParams->XorMask)
1672 OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: XorMask));
1673 return OffsetLong;
1674 }
1675
1676 /// Compute the shadow and origin addresses corresponding to a given
1677 /// application address.
1678 ///
1679 /// Shadow = ShadowBase + Offset
1680 /// Origin = (OriginBase + Offset) & ~3ULL
1681 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1682 /// a single pointee.
1683 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1684 std::pair<Value *, Value *>
1685 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1686 MaybeAlign Alignment) {
1687 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1688 if (!VectTy) {
1689 assert(Addr->getType()->isPointerTy());
1690 } else {
1691 assert(VectTy->getElementType()->isPointerTy());
1692 }
1693 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1694 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1695 Value *ShadowLong = ShadowOffset;
1696 if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1697 ShadowLong =
1698 IRB.CreateAdd(LHS: ShadowLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ShadowBase));
1699 }
1700 Value *ShadowPtr = IRB.CreateIntToPtr(
1701 V: ShadowLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy));
1702
1703 Value *OriginPtr = nullptr;
1704 if (MS.TrackOrigins) {
1705 Value *OriginLong = ShadowOffset;
1706 uint64_t OriginBase = MS.MapParams->OriginBase;
1707 if (OriginBase != 0)
1708 OriginLong =
1709 IRB.CreateAdd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: OriginBase));
1710 if (!Alignment || *Alignment < kMinOriginAlignment) {
1711 uint64_t Mask = kMinOriginAlignment.value() - 1;
1712 OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~Mask));
1713 }
1714 OriginPtr = IRB.CreateIntToPtr(
1715 V: OriginLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy: MS.OriginTy));
1716 }
1717 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1718 }
1719
1720 template <typename... ArgsTy>
1721 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1722 ArgsTy... Args) {
1723 if (MS.TargetTriple.getArch() == Triple::systemz) {
1724 IRB.CreateCall(Callee,
1725 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1726 return IRB.CreateLoad(Ty: MS.MsanMetadata, Ptr: MS.MsanMetadataAlloca);
1727 }
1728
1729 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1730 }
1731
1732 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
1733 IRBuilder<> &IRB,
1734 Type *ShadowTy,
1735 bool isStore) {
1736 Value *ShadowOriginPtrs;
1737 const DataLayout &DL = F.getDataLayout();
1738 TypeSize Size = DL.getTypeStoreSize(Ty: ShadowTy);
1739
1740 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, size: Size);
1741 Value *AddrCast = IRB.CreatePointerCast(V: Addr, DestTy: MS.PtrTy);
1742 if (Getter) {
1743 ShadowOriginPtrs = createMetadataCall(IRB, Callee: Getter, Args: AddrCast);
1744 } else {
1745 Value *SizeVal = ConstantInt::get(Ty: MS.IntptrTy, V: Size);
1746 ShadowOriginPtrs = createMetadataCall(
1747 IRB,
1748 Callee: isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1749 Args: AddrCast, Args: SizeVal);
1750 }
1751 Value *ShadowPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 0);
1752 ShadowPtr = IRB.CreatePointerCast(V: ShadowPtr, DestTy: MS.PtrTy);
1753 Value *OriginPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 1);
1754
1755 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1756 }
1757
1758 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1759 /// a single pointee.
1760 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1761 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1762 IRBuilder<> &IRB,
1763 Type *ShadowTy,
1764 bool isStore) {
1765 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1766 if (!VectTy) {
1767 assert(Addr->getType()->isPointerTy());
1768 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1769 }
1770
1771 // TODO: Support callbacs with vectors of addresses.
1772 unsigned NumElements = cast<FixedVectorType>(Val: VectTy)->getNumElements();
1773 Value *ShadowPtrs = ConstantInt::getNullValue(
1774 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1775 Value *OriginPtrs = nullptr;
1776 if (MS.TrackOrigins)
1777 OriginPtrs = ConstantInt::getNullValue(
1778 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1779 for (unsigned i = 0; i < NumElements; ++i) {
1780 Value *OneAddr =
1781 IRB.CreateExtractElement(Vec: Addr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1782 auto [ShadowPtr, OriginPtr] =
1783 getShadowOriginPtrKernelNoVec(Addr: OneAddr, IRB, ShadowTy, isStore);
1784
1785 ShadowPtrs = IRB.CreateInsertElement(
1786 Vec: ShadowPtrs, NewElt: ShadowPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1787 if (MS.TrackOrigins)
1788 OriginPtrs = IRB.CreateInsertElement(
1789 Vec: OriginPtrs, NewElt: OriginPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1790 }
1791 return {ShadowPtrs, OriginPtrs};
1792 }
1793
1794 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1795 Type *ShadowTy,
1796 MaybeAlign Alignment,
1797 bool isStore) {
1798 if (MS.CompileKernel)
1799 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1800 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1801 }
1802
1803 /// Compute the shadow address for a given function argument.
1804 ///
1805 /// Shadow = ParamTLS+ArgOffset.
1806 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1807 return IRB.CreatePtrAdd(Ptr: MS.ParamTLS,
1808 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg");
1809 }
1810
1811 /// Compute the origin address for a given function argument.
1812 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1813 if (!MS.TrackOrigins)
1814 return nullptr;
1815 return IRB.CreatePtrAdd(Ptr: MS.ParamOriginTLS,
1816 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
1817 Name: "_msarg_o");
1818 }
1819
1820 /// Compute the shadow address for a retval.
1821 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1822 return IRB.CreatePointerCast(V: MS.RetvalTLS, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msret");
1823 }
1824
1825 /// Compute the origin address for a retval.
1826 Value *getOriginPtrForRetval() {
1827 // We keep a single origin for the entire retval. Might be too optimistic.
1828 return MS.RetvalOriginTLS;
1829 }
1830
1831 /// Set SV to be the shadow value for V.
1832 void setShadow(Value *V, Value *SV) {
1833 assert(!ShadowMap.count(V) && "Values may only have one shadow");
1834
1835 // getShadowTy(VoidType) == nullptr
1836 if (SV && !V->getType()->isVoidTy() && getShadowTy(V) != SV->getType()) {
1837 LLVM_DEBUG(dbgs() << "V: " << *V << "\n");
1838 LLVM_DEBUG(dbgs() << "SV: " << *SV << "\n");
1839 assert(getShadowTy(V) == SV->getType() &&
1840 "setShadow called with mismatching types");
1841 }
1842
1843 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1844 }
1845
1846 /// Set Origin to be the origin value for V.
1847 void setOrigin(Value *V, Value *Origin) {
1848 if (!MS.TrackOrigins)
1849 return;
1850 assert(!OriginMap.count(V) && "Values may only have one origin");
1851 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1852 OriginMap[V] = Origin;
1853 }
1854
1855 Constant *getCleanShadow(Type *OrigTy) {
1856 Type *ShadowTy = getShadowTy(OrigTy);
1857 if (!ShadowTy)
1858 return nullptr;
1859 return Constant::getNullValue(Ty: ShadowTy);
1860 }
1861
1862 /// Create a clean shadow value for a given value.
1863 ///
1864 /// Clean shadow (all zeroes) means all bits of the value are defined
1865 /// (initialized).
1866 Constant *getCleanShadow(Value *V) { return getCleanShadow(OrigTy: V->getType()); }
1867
1868 /// Create a dirty shadow of a given shadow type.
1869 Constant *getPoisonedShadow(Type *ShadowTy) {
1870 assert(ShadowTy);
1871 if (isa<IntegerType>(Val: ShadowTy) || isa<VectorType>(Val: ShadowTy))
1872 return Constant::getAllOnesValue(Ty: ShadowTy);
1873 if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy)) {
1874 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1875 getPoisonedShadow(ShadowTy: AT->getElementType()));
1876 return ConstantArray::get(T: AT, V: Vals);
1877 }
1878 if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy)) {
1879 SmallVector<Constant *, 4> Vals;
1880 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1881 Vals.push_back(Elt: getPoisonedShadow(ShadowTy: ST->getElementType(N: i)));
1882 return ConstantStruct::get(T: ST, V: Vals);
1883 }
1884 llvm_unreachable("Unexpected shadow type");
1885 }
1886
1887 /// Create a dirty shadow for a given value.
1888 Constant *getPoisonedShadow(Value *V) {
1889 Type *ShadowTy = getShadowTy(V);
1890 if (!ShadowTy)
1891 return nullptr;
1892 return getPoisonedShadow(ShadowTy);
1893 }
1894
1895 /// Create a clean (zero) origin.
1896 Value *getCleanOrigin() { return Constant::getNullValue(Ty: MS.OriginTy); }
1897
1898 /// Get the shadow value for a given Value.
1899 ///
1900 /// This function either returns the value set earlier with setShadow,
1901 /// or extracts if from ParamTLS (for function arguments).
1902 Value *getShadow(Value *V) {
1903 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
1904 if (!PropagateShadow || I->getMetadata(KindID: LLVMContext::MD_nosanitize))
1905 return getCleanShadow(V);
1906 // For instructions the shadow is already stored in the map.
1907 Value *Shadow = ShadowMap[V];
1908 if (!Shadow) {
1909 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1910 assert(Shadow && "No shadow for a value");
1911 }
1912 return Shadow;
1913 }
1914 // Handle fully undefined values
1915 // (partially undefined constant vectors are handled later)
1916 if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(Val: V)) {
1917 Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1918 : getCleanShadow(V);
1919 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1920 return AllOnes;
1921 }
1922 if (Argument *A = dyn_cast<Argument>(Val: V)) {
1923 // For arguments we compute the shadow on demand and store it in the map.
1924 Value *&ShadowPtr = ShadowMap[V];
1925 if (ShadowPtr)
1926 return ShadowPtr;
1927 Function *F = A->getParent();
1928 IRBuilder<> EntryIRB(FnPrologueEnd);
1929 unsigned ArgOffset = 0;
1930 const DataLayout &DL = F->getDataLayout();
1931 for (auto &FArg : F->args()) {
1932 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1933 LLVM_DEBUG(dbgs() << (FArg.getType()->isScalableTy()
1934 ? "vscale not fully supported\n"
1935 : "Arg is not sized\n"));
1936 if (A == &FArg) {
1937 ShadowPtr = getCleanShadow(V);
1938 setOrigin(V: A, Origin: getCleanOrigin());
1939 break;
1940 }
1941 continue;
1942 }
1943
1944 unsigned Size = FArg.hasByValAttr()
1945 ? DL.getTypeAllocSize(Ty: FArg.getParamByValType())
1946 : DL.getTypeAllocSize(Ty: FArg.getType());
1947
1948 if (A == &FArg) {
1949 bool Overflow = ArgOffset + Size > kParamTLSSize;
1950 if (FArg.hasByValAttr()) {
1951 // ByVal pointer itself has clean shadow. We copy the actual
1952 // argument shadow to the underlying memory.
1953 // Figure out maximal valid memcpy alignment.
1954 const Align ArgAlign = DL.getValueOrABITypeAlignment(
1955 Alignment: FArg.getParamAlign(), Ty: FArg.getParamByValType());
1956 Value *CpShadowPtr, *CpOriginPtr;
1957 std::tie(args&: CpShadowPtr, args&: CpOriginPtr) =
1958 getShadowOriginPtr(Addr: V, IRB&: EntryIRB, ShadowTy: EntryIRB.getInt8Ty(), Alignment: ArgAlign,
1959 /*isStore*/ true);
1960 if (!PropagateShadow || Overflow) {
1961 // ParamTLS overflow.
1962 EntryIRB.CreateMemSet(
1963 Ptr: CpShadowPtr, Val: Constant::getNullValue(Ty: EntryIRB.getInt8Ty()),
1964 Size, Align: ArgAlign);
1965 } else {
1966 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
1967 const Align CopyAlign = std::min(a: ArgAlign, b: kShadowTLSAlignment);
1968 [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
1969 Dst: CpShadowPtr, DstAlign: CopyAlign, Src: Base, SrcAlign: CopyAlign, Size);
1970 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1971
1972 if (MS.TrackOrigins) {
1973 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
1974 // FIXME: OriginSize should be:
1975 // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1976 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
1977 EntryIRB.CreateMemCpy(
1978 Dst: CpOriginPtr,
1979 /* by getShadowOriginPtr */ DstAlign: kMinOriginAlignment, Src: OriginPtr,
1980 /* by origin_tls[ArgOffset] */ SrcAlign: kMinOriginAlignment,
1981 Size: OriginSize);
1982 }
1983 }
1984 }
1985
1986 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1987 (MS.EagerChecks && FArg.hasAttribute(Kind: Attribute::NoUndef))) {
1988 ShadowPtr = getCleanShadow(V);
1989 setOrigin(V: A, Origin: getCleanOrigin());
1990 } else {
1991 // Shadow over TLS
1992 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
1993 ShadowPtr = EntryIRB.CreateAlignedLoad(Ty: getShadowTy(V: &FArg), Ptr: Base,
1994 Align: kShadowTLSAlignment);
1995 if (MS.TrackOrigins) {
1996 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
1997 setOrigin(V: A, Origin: EntryIRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
1998 }
1999 }
2000 LLVM_DEBUG(dbgs()
2001 << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
2002 break;
2003 }
2004
2005 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
2006 }
2007 assert(ShadowPtr && "Could not find shadow for an argument");
2008 return ShadowPtr;
2009 }
2010
2011 // Check for partially-undefined constant vectors
2012 // TODO: scalable vectors (this is hard because we do not have IRBuilder)
2013 if (isa<FixedVectorType>(Val: V->getType()) && isa<Constant>(Val: V) &&
2014 cast<Constant>(Val: V)->containsUndefOrPoisonElement() && PropagateShadow &&
2015 PoisonUndefVectors) {
2016 unsigned NumElems = cast<FixedVectorType>(Val: V->getType())->getNumElements();
2017 SmallVector<Constant *, 32> ShadowVector(NumElems);
2018 for (unsigned i = 0; i != NumElems; ++i) {
2019 Constant *Elem = cast<Constant>(Val: V)->getAggregateElement(Elt: i);
2020 ShadowVector[i] = isa<UndefValue>(Val: Elem) ? getPoisonedShadow(V: Elem)
2021 : getCleanShadow(V: Elem);
2022 }
2023
2024 Value *ShadowConstant = ConstantVector::get(V: ShadowVector);
2025 LLVM_DEBUG(dbgs() << "Partial undef constant vector: " << *V << " ==> "
2026 << *ShadowConstant << "\n");
2027
2028 return ShadowConstant;
2029 }
2030
2031 // TODO: partially-undefined constant arrays, structures, and nested types
2032
2033 // For everything else the shadow is zero.
2034 return getCleanShadow(V);
2035 }
2036
2037 /// Get the shadow for i-th argument of the instruction I.
2038 Value *getShadow(Instruction *I, int i) {
2039 return getShadow(V: I->getOperand(i));
2040 }
2041
2042 /// Get the origin for a value.
2043 Value *getOrigin(Value *V) {
2044 if (!MS.TrackOrigins)
2045 return nullptr;
2046 if (!PropagateShadow || isa<Constant>(Val: V) || isa<InlineAsm>(Val: V))
2047 return getCleanOrigin();
2048 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2049 "Unexpected value type in getOrigin()");
2050 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2051 if (I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2052 return getCleanOrigin();
2053 }
2054 Value *Origin = OriginMap[V];
2055 assert(Origin && "Missing origin");
2056 return Origin;
2057 }
2058
2059 /// Get the origin for i-th argument of the instruction I.
2060 Value *getOrigin(Instruction *I, int i) {
2061 return getOrigin(V: I->getOperand(i));
2062 }
2063
2064 /// Remember the place where a shadow check should be inserted.
2065 ///
2066 /// This location will be later instrumented with a check that will print a
2067 /// UMR warning in runtime if the shadow value is not 0.
2068 void insertCheckShadow(Value *Shadow, Value *Origin, Instruction *OrigIns) {
2069 assert(Shadow);
2070 if (!InsertChecks)
2071 return;
2072
2073 if (!DebugCounter::shouldExecute(Counter&: DebugInsertCheck)) {
2074 LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
2075 << *OrigIns << "\n");
2076 return;
2077 }
2078
2079 Type *ShadowTy = Shadow->getType();
2080 if (isScalableNonVectorType(Ty: ShadowTy)) {
2081 LLVM_DEBUG(dbgs() << "Skipping check of scalable non-vector " << *Shadow
2082 << " before " << *OrigIns << "\n");
2083 return;
2084 }
2085#ifndef NDEBUG
2086 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2087 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2088 "Can only insert checks for integer, vector, and aggregate shadow "
2089 "types");
2090#endif
2091 InstrumentationList.push_back(
2092 Elt: ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2093 }
2094
2095 /// Get shadow for value, and remember the place where a shadow check should
2096 /// be inserted.
2097 ///
2098 /// This location will be later instrumented with a check that will print a
2099 /// UMR warning in runtime if the value is not fully defined.
2100 void insertCheckShadowOf(Value *Val, Instruction *OrigIns) {
2101 assert(Val);
2102 Value *Shadow, *Origin;
2103 if (Opts.msan_check_constant_shadow) {
2104 Shadow = getShadow(V: Val);
2105 if (!Shadow)
2106 return;
2107 Origin = getOrigin(V: Val);
2108 } else {
2109 Shadow = dyn_cast_or_null<Instruction>(Val: getShadow(V: Val));
2110 if (!Shadow)
2111 return;
2112 Origin = dyn_cast_or_null<Instruction>(Val: getOrigin(V: Val));
2113 }
2114 insertCheckShadow(Shadow, Origin, OrigIns);
2115 }
2116
2117 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
2118 switch (a) {
2119 case AtomicOrdering::NotAtomic:
2120 return AtomicOrdering::NotAtomic;
2121 case AtomicOrdering::Unordered:
2122 case AtomicOrdering::Monotonic:
2123 case AtomicOrdering::Release:
2124 return AtomicOrdering::Release;
2125 case AtomicOrdering::Acquire:
2126 case AtomicOrdering::AcquireRelease:
2127 return AtomicOrdering::AcquireRelease;
2128 case AtomicOrdering::SequentiallyConsistent:
2129 return AtomicOrdering::SequentiallyConsistent;
2130 }
2131 llvm_unreachable("Unknown ordering");
2132 }
2133
2134 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2135 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2136 uint32_t OrderingTable[NumOrderings] = {};
2137
2138 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2139 OrderingTable[(int)AtomicOrderingCABI::release] =
2140 (int)AtomicOrderingCABI::release;
2141 OrderingTable[(int)AtomicOrderingCABI::consume] =
2142 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2143 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2144 (int)AtomicOrderingCABI::acq_rel;
2145 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2146 (int)AtomicOrderingCABI::seq_cst;
2147
2148 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2149 }
2150
2151 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
2152 switch (a) {
2153 case AtomicOrdering::NotAtomic:
2154 return AtomicOrdering::NotAtomic;
2155 case AtomicOrdering::Unordered:
2156 case AtomicOrdering::Monotonic:
2157 case AtomicOrdering::Acquire:
2158 return AtomicOrdering::Acquire;
2159 case AtomicOrdering::Release:
2160 case AtomicOrdering::AcquireRelease:
2161 return AtomicOrdering::AcquireRelease;
2162 case AtomicOrdering::SequentiallyConsistent:
2163 return AtomicOrdering::SequentiallyConsistent;
2164 }
2165 llvm_unreachable("Unknown ordering");
2166 }
2167
2168 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2169 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2170 uint32_t OrderingTable[NumOrderings] = {};
2171
2172 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2173 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2174 OrderingTable[(int)AtomicOrderingCABI::consume] =
2175 (int)AtomicOrderingCABI::acquire;
2176 OrderingTable[(int)AtomicOrderingCABI::release] =
2177 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2178 (int)AtomicOrderingCABI::acq_rel;
2179 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2180 (int)AtomicOrderingCABI::seq_cst;
2181
2182 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2183 }
2184
2185 // ------------------- Visitors.
2186 using InstVisitor<MemorySanitizerVisitor>::visit;
2187 void visit(Instruction &I) {
2188 if (I.getMetadata(KindID: LLVMContext::MD_nosanitize))
2189 return;
2190 // Don't want to visit if we're in the prologue
2191 if (isInPrologue(I))
2192 return;
2193 if (!DebugCounter::shouldExecute(Counter&: DebugInstrumentInstruction)) {
2194 LLVM_DEBUG(dbgs() << "Skipping instruction: " << I << "\n");
2195 // We still need to set the shadow and origin to clean values.
2196 setShadow(V: &I, SV: getCleanShadow(V: &I));
2197 setOrigin(V: &I, Origin: getCleanOrigin());
2198 return;
2199 }
2200
2201 Instructions.push_back(Elt: &I);
2202 }
2203
2204 /// Instrument LoadInst
2205 ///
2206 /// Loads the corresponding shadow and (optionally) origin.
2207 /// Optionally, checks that the load address is fully defined.
2208 void visitLoadInst(LoadInst &I) {
2209 assert(I.getType()->isSized() && "Load type must have size");
2210 assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2211 NextNodeIRBuilder IRB(&I);
2212 Type *ShadowTy = getShadowTy(V: &I);
2213 Value *Addr = I.getPointerOperand();
2214 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2215 const Align Alignment = I.getAlign();
2216 if (PropagateShadow) {
2217 std::tie(args&: ShadowPtr, args&: OriginPtr) =
2218 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2219 setShadow(V: &I,
2220 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2221 } else {
2222 setShadow(V: &I, SV: getCleanShadow(V: &I));
2223 }
2224
2225 if (Opts.msan_check_access_address)
2226 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2227
2228 if (I.isAtomic())
2229 I.setOrdering(addAcquireOrdering(a: I.getOrdering()));
2230
2231 if (MS.TrackOrigins) {
2232 if (PropagateShadow) {
2233 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
2234 setOrigin(
2235 V: &I, Origin: IRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: OriginPtr, Align: OriginAlignment));
2236 } else {
2237 setOrigin(V: &I, Origin: getCleanOrigin());
2238 }
2239 }
2240 }
2241
2242 /// Instrument StoreInst
2243 ///
2244 /// Stores the corresponding shadow and (optionally) origin.
2245 /// Optionally, checks that the store address is fully defined.
2246 void visitStoreInst(StoreInst &I) {
2247 StoreList.push_back(Elt: &I);
2248 if (Opts.msan_check_access_address)
2249 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2250 }
2251
2252 void handleCASOrRMW(Instruction &I) {
2253 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2254
2255 IRBuilder<> IRB(&I);
2256 Value *Addr = I.getOperand(i: 0);
2257 Value *Val = I.getOperand(i: 1);
2258 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, ShadowTy: getShadowTy(V: Val), Alignment: Align(1),
2259 /*isStore*/ true)
2260 .first;
2261
2262 if (Opts.msan_check_access_address)
2263 insertCheckShadowOf(Val: Addr, OrigIns: &I);
2264
2265 // Only test the conditional argument of cmpxchg instruction.
2266 // The other argument can potentially be uninitialized, but we can not
2267 // detect this situation reliably without possible false positives.
2268 if (isa<AtomicCmpXchgInst>(Val: I))
2269 insertCheckShadowOf(Val, OrigIns: &I);
2270
2271 IRB.CreateStore(Val: getCleanShadow(V: Val), Ptr: ShadowPtr);
2272
2273 setShadow(V: &I, SV: getCleanShadow(V: &I));
2274 setOrigin(V: &I, Origin: getCleanOrigin());
2275 }
2276
2277 void visitAtomicRMWInst(AtomicRMWInst &I) {
2278 handleCASOrRMW(I);
2279 I.setOrdering(addReleaseOrdering(a: I.getOrdering()));
2280 }
2281
2282 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2283 handleCASOrRMW(I);
2284 I.setSuccessOrdering(addReleaseOrdering(a: I.getSuccessOrdering()));
2285 }
2286
2287 /// Generic handler to compute shadow for == and != comparisons.
2288 ///
2289 /// This function is used by handleEqualityComparison and visitSwitchInst.
2290 ///
2291 /// Sometimes the comparison result is known even if some of the bits of the
2292 /// arguments are not.
2293 Value *propagateEqualityComparison(IRBuilder<> &IRB, Value *A, Value *B,
2294 Value *Sa, Value *Sb) {
2295 assert(getShadowTy(A) == Sa->getType());
2296 assert(getShadowTy(B) == Sb->getType());
2297
2298 // Get rid of pointers and vectors of pointers.
2299 // For ints (and vectors of ints), types of A and Sa match,
2300 // and this is a no-op.
2301 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2302 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2303
2304 // A == B <==> (C = A^B) == 0
2305 // A != B <==> (C = A^B) != 0
2306 // Sc = Sa | Sb
2307 Value *C = IRB.CreateXor(LHS: A, RHS: B);
2308 Value *Sc = IRB.CreateOr(LHS: Sa, RHS: Sb);
2309 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2310 // Result is defined if one of the following is true
2311 // * there is a defined 1 bit in C
2312 // * C is fully defined
2313 // Si = !(C & ~Sc) && Sc
2314 Value *Zero = Constant::getNullValue(Ty: Sc->getType());
2315 Value *MinusOne = Constant::getAllOnesValue(Ty: Sc->getType());
2316 Value *LHS = IRB.CreateICmpNE(LHS: Sc, RHS: Zero);
2317 Value *RHS =
2318 IRB.CreateICmpEQ(LHS: IRB.CreateAnd(LHS: IRB.CreateXor(LHS: Sc, RHS: MinusOne), RHS: C), RHS: Zero);
2319 Value *Si = IRB.CreateAnd(LHS, RHS);
2320 Si->setName("_msprop_icmp");
2321
2322 return Si;
2323 }
2324
2325 // Instrument:
2326 // switch i32 %Val, label %else [ i32 0, label %A
2327 // i32 1, label %B
2328 // i32 2, label %C ]
2329 //
2330 // Typically, the switch input value (%Val) is fully initialized.
2331 //
2332 // Sometimes the compiler may convert (icmp + br) into a switch statement.
2333 // MSan allows icmp eq/ne with partly initialized inputs to still result in a
2334 // fully initialized output, if there exists a bit that is initialized in
2335 // both inputs with a differing value. For compatibility, we support this in
2336 // the switch instrumentation as well. Note that this edge case only applies
2337 // if the switch input value does not match *any* of the cases (matching any
2338 // of the cases requires an exact, fully initialized match).
2339 //
2340 // ShadowCases = 0
2341 // | propagateEqualityComparison(Val, 0)
2342 // | propagateEqualityComparison(Val, 1)
2343 // | propagateEqualityComparison(Val, 2))
2344 void visitSwitchInst(SwitchInst &SI) {
2345 IRBuilder<> IRB(&SI);
2346
2347 Value *Val = SI.getCondition();
2348 Value *ShadowVal = getShadow(V: Val);
2349 // TODO: add fast path - if the condition is fully initialized, we know
2350 // there is no UUM, without needing to consider the case values below.
2351
2352 // Some code (e.g., AMDGPUGenMCCodeEmitter.inc) has tens of thousands of
2353 // cases. This results in an extremely long chained expression for MSan's
2354 // switch instrumentation, which can cause the JumpThreadingPass to have a
2355 // stack overflow or excessive runtime. We limit the number of cases
2356 // considered, with the tradeoff of niche false negatives.
2357 // TODO: figure out a better solution.
2358 int casesToConsider = Opts.msan_switch_precision;
2359
2360 Value *ShadowCases = nullptr;
2361 for (auto Case : SI.cases()) {
2362 if (casesToConsider <= 0)
2363 break;
2364
2365 Value *Comparator = Case.getCaseValue();
2366 // TODO: some simplification is possible when comparing multiple cases
2367 // simultaneously.
2368 Value *ComparisonShadow = propagateEqualityComparison(
2369 IRB, A: Val, B: Comparator, Sa: ShadowVal, Sb: getShadow(V: Comparator));
2370
2371 if (ShadowCases)
2372 ShadowCases = IRB.CreateOr(LHS: ShadowCases, RHS: ComparisonShadow);
2373 else
2374 ShadowCases = ComparisonShadow;
2375
2376 casesToConsider--;
2377 }
2378
2379 if (ShadowCases)
2380 insertCheckShadow(Shadow: ShadowCases, Origin: getOrigin(V: Val), OrigIns: &SI);
2381 }
2382
2383 // Vector manipulation.
2384 void visitExtractElementInst(ExtractElementInst &I) {
2385 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 1), OrigIns: &I);
2386 IRBuilder<> IRB(&I);
2387 setShadow(V: &I, SV: IRB.CreateExtractElement(Vec: getShadow(I: &I, i: 0), Idx: I.getOperand(i_nocapture: 1),
2388 Name: "_msprop"));
2389 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2390 }
2391
2392 void visitInsertElementInst(InsertElementInst &I) {
2393 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 2), OrigIns: &I);
2394 IRBuilder<> IRB(&I);
2395 auto *Shadow0 = getShadow(I: &I, i: 0);
2396 auto *Shadow1 = getShadow(I: &I, i: 1);
2397 setShadow(V: &I, SV: IRB.CreateInsertElement(Vec: Shadow0, NewElt: Shadow1, Idx: I.getOperand(i_nocapture: 2),
2398 Name: "_msprop"));
2399 setOriginForNaryOp(I);
2400 }
2401
2402 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2403 IRBuilder<> IRB(&I);
2404 auto *Shadow0 = getShadow(I: &I, i: 0);
2405 auto *Shadow1 = getShadow(I: &I, i: 1);
2406 setShadow(V: &I, SV: IRB.CreateShuffleVector(V1: Shadow0, V2: Shadow1, Mask: I.getShuffleMask(),
2407 Name: "_msprop"));
2408 setOriginForNaryOp(I);
2409 }
2410
2411 // Casts.
2412 void visitSExtInst(SExtInst &I) {
2413 IRBuilder<> IRB(&I);
2414 setShadow(V: &I, SV: IRB.CreateSExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2415 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2416 }
2417
2418 void visitZExtInst(ZExtInst &I) {
2419 IRBuilder<> IRB(&I);
2420 setShadow(V: &I, SV: IRB.CreateZExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2421 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2422 }
2423
2424 void visitTruncInst(TruncInst &I) {
2425 IRBuilder<> IRB(&I);
2426 setShadow(V: &I, SV: IRB.CreateTrunc(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2427 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2428 }
2429
2430 void visitBitCastInst(BitCastInst &I) {
2431 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2432 // a musttail call and a ret, don't instrument. New instructions are not
2433 // allowed after a musttail call.
2434 if (auto *CI = dyn_cast<CallInst>(Val: I.getOperand(i_nocapture: 0)))
2435 if (CI->isMustTailCall())
2436 return;
2437 IRBuilder<> IRB(&I);
2438 setShadow(V: &I, SV: IRB.CreateBitCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I)));
2439 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2440 }
2441
2442 void visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
2443 IRBuilder<> IRB(&I);
2444 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2445 Name: "_msprop_addrspacecast"));
2446 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2447 }
2448
2449 void visitPtrToIntInst(PtrToIntInst &I) {
2450 IRBuilder<> IRB(&I);
2451 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2452 Name: "_msprop_ptrtoint"));
2453 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2454 }
2455
2456 void visitPtrToAddrInst(PtrToAddrInst &I) {
2457 IRBuilder<> IRB(&I);
2458 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2459 Name: "_msprop_ptrtoaddr"));
2460 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2461 }
2462
2463 void visitIntToPtrInst(IntToPtrInst &I) {
2464 IRBuilder<> IRB(&I);
2465 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2466 Name: "_msprop_inttoptr"));
2467 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2468 }
2469
2470 /// Handle LLVM and NEON vector convert intrinsics.
2471 ///
2472 /// e.g., <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>)
2473 /// i32 @llvm.aarch64.neon.fcvtms.i32.f64 (double)
2474 /// <2 x i32> @fptoui (<2 x float>)
2475 /// i64 @llvm.fptosi.sat.i64.f64(double)
2476 ///
2477 /// Note that the size of input/output elements can differ e.g.,
2478 /// double @sitofp(i32)
2479 /// but the number of elements must be the same.
2480 ///
2481 /// For conversions to or from fixed-point, there is a trailing argument to
2482 /// indicate the fixed-point precision:
2483 /// - <4 x float> llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32>, i32)
2484 /// - <4 x i32> llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float>, i32)
2485 ///
2486 /// For x86 SSE vector convert intrinsics, see
2487 /// handleSSEVectorConvertIntrinsic().
2488 void handleGenericVectorConvertIntrinsic(Instruction &I, bool FixedPoint) {
2489 [[maybe_unused]] unsigned NumArgs = I.getNumOperands();
2490 if (auto *CI = dyn_cast<CallInst>(Val: &I))
2491 NumArgs = CI->arg_size();
2492
2493 if (FixedPoint) {
2494 assert(NumArgs == 2);
2495 Value *Precision = I.getOperand(i: 1);
2496 insertCheckShadowOf(Val: Precision, OrigIns: &I);
2497 } else {
2498 assert(NumArgs == 1);
2499 }
2500
2501 IRBuilder<> IRB(&I);
2502 Value *S0 = getShadow(I: &I, i: 0);
2503
2504 /// For scalars:
2505 /// Since they are converting from floating-point to integer, or between
2506 /// different width floating-point values, the output is:
2507 /// - fully uninitialized if *any* bit of the input is uninitialized
2508 /// - fully ininitialized if all bits of the input are ininitialized
2509 /// We apply the same principle on a per-field basis for vectors.
2510 Value *OutShadow = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)),
2511 DestTy: getShadowTy(V: &I));
2512 setShadow(V: &I, SV: OutShadow);
2513 setOriginForNaryOp(I);
2514 }
2515
2516 void visitFPToSIInst(CastInst &I) {
2517 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2518 }
2519 void visitFPToUIInst(CastInst &I) {
2520 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2521 }
2522 void visitSIToFPInst(CastInst &I) {
2523 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2524 }
2525 void visitUIToFPInst(CastInst &I) {
2526 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2527 }
2528
2529 void visitFPExtInst(CastInst &I) {
2530 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2531 }
2532 void visitFPTruncInst(CastInst &I) {
2533 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2534 }
2535
2536 /// Generic handler to compute shadow for bitwise AND.
2537 ///
2538 /// This is used by 'visitAnd' but also as a primitive for other handlers.
2539 ///
2540 /// This code is precise: it implements the rule that "And" of an initialized
2541 /// zero bit always results in an initialized value:
2542 // 1&1 => 1; 0&1 => 0; p&1 => p;
2543 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2544 // 1&p => p; 0&p => 0; p&p => p;
2545 //
2546 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2547 Value *handleBitwiseAnd(IRBuilder<> &IRB, Value *V1, Value *V2, Value *S1,
2548 Value *S2) {
2549 // "The two arguments to the ‘and’ instruction must be integer or vector
2550 // of integer values. Both arguments must have identical types."
2551 //
2552 // We enforce this condition for all callers to handleBitwiseAnd(); callers
2553 // with non-integer types should call CreateAppToShadowCast() themselves.
2554 assert(V1->getType()->isIntOrIntVectorTy());
2555 assert(V1->getType() == V2->getType());
2556
2557 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2558 assert(V1->getType() == S1->getType());
2559 assert(V2->getType() == S2->getType());
2560
2561 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2562 Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2563 Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2564
2565 return IRB.CreateOr(Ops: {S1S2, V1S2, S1V2});
2566 }
2567
2568 /// Handler for bitwise AND operator.
2569 void visitAnd(BinaryOperator &I) {
2570 IRBuilder<> IRB(&I);
2571 Value *V1 = I.getOperand(i_nocapture: 0);
2572 Value *V2 = I.getOperand(i_nocapture: 1);
2573 Value *S1 = getShadow(I: &I, i: 0);
2574 Value *S2 = getShadow(I: &I, i: 1);
2575
2576 Value *OutShadow = handleBitwiseAnd(IRB, V1, V2, S1, S2);
2577
2578 setShadow(V: &I, SV: OutShadow);
2579 setOriginForNaryOp(I);
2580 }
2581
2582 void visitOr(BinaryOperator &I) {
2583 IRBuilder<> IRB(&I);
2584 // "Or" of 1 and a poisoned value results in unpoisoned value:
2585 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2586 // 1|0 => 1; 0|0 => 0; p|0 => p;
2587 // 1|p => 1; 0|p => p; p|p => p;
2588 //
2589 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2590 //
2591 // If the "disjoint OR" property is violated, the result is poison, and
2592 // hence the entire shadow is uninitialized:
2593 // S = S | SignExt(V1 & V2 != 0)
2594 Value *S1 = getShadow(I: &I, i: 0);
2595 Value *S2 = getShadow(I: &I, i: 1);
2596 Value *V1 = I.getOperand(i_nocapture: 0);
2597 Value *V2 = I.getOperand(i_nocapture: 1);
2598
2599 // "The two arguments to the ‘or’ instruction must be integer or vector
2600 // of integer values. Both arguments must have identical types."
2601 assert(V1->getType()->isIntOrIntVectorTy());
2602 assert(V1->getType() == V2->getType());
2603
2604 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2605 assert(V1->getType() == S1->getType());
2606 assert(V2->getType() == S2->getType());
2607
2608 Value *NotV1 = IRB.CreateNot(V: V1);
2609 Value *NotV2 = IRB.CreateNot(V: V2);
2610
2611 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2612 Value *S2NotV1 = IRB.CreateAnd(LHS: NotV1, RHS: S2);
2613 Value *S1NotV2 = IRB.CreateAnd(LHS: S1, RHS: NotV2);
2614
2615 Value *S = IRB.CreateOr(Ops: {S1S2, S2NotV1, S1NotV2});
2616
2617 if (Opts.msan_precise_disjoint_or &&
2618 cast<PossiblyDisjointInst>(Val: &I)->isDisjoint()) {
2619 Value *V1V2 = IRB.CreateAnd(LHS: V1, RHS: V2);
2620 Value *DisjointOrShadow = IRB.CreateSExt(
2621 V: IRB.CreateICmpNE(LHS: V1V2, RHS: getCleanShadow(V: V1V2)), DestTy: V1V2->getType());
2622 S = IRB.CreateOr(LHS: S, RHS: DisjointOrShadow, Name: "_ms_disjoint");
2623 }
2624
2625 setShadow(V: &I, SV: S);
2626 setOriginForNaryOp(I);
2627 }
2628
2629 /// Default propagation of shadow and/or origin.
2630 ///
2631 /// This class implements the general case of shadow propagation, used in all
2632 /// cases where we don't know and/or don't care about what the operation
2633 /// actually does. It converts all input shadow values to a common type
2634 /// (extending or truncating as necessary), and bitwise OR's them.
2635 ///
2636 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2637 /// fully initialized), and less prone to false positives.
2638 ///
2639 /// This class also implements the general case of origin propagation. For a
2640 /// Nary operation, result origin is set to the origin of an argument that is
2641 /// not entirely initialized. If there is more than one such arguments, the
2642 /// rightmost of them is picked. It does not matter which one is picked if all
2643 /// arguments are initialized.
2644 template <bool CombineShadow> class Combiner {
2645 Value *Shadow = nullptr;
2646 Value *Origin = nullptr;
2647 IRBuilder<> &IRB;
2648 MemorySanitizerVisitor *MSV;
2649
2650 public:
2651 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2652 : IRB(IRB), MSV(MSV) {}
2653
2654 /// Add a pair of shadow and origin values to the mix.
2655 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2656 if (CombineShadow) {
2657 assert(OpShadow);
2658 if (!Shadow)
2659 Shadow = OpShadow;
2660 else {
2661 OpShadow = MSV->CreateShadowCast(IRB, V: OpShadow, dstTy: Shadow->getType());
2662 Shadow = IRB.CreateOr(LHS: Shadow, RHS: OpShadow, Name: "_msprop");
2663 }
2664 }
2665
2666 if (MSV->MS.TrackOrigins) {
2667 assert(OpOrigin);
2668 if (!Origin) {
2669 Origin = OpOrigin;
2670 } else {
2671 Constant *ConstOrigin = dyn_cast<Constant>(Val: OpOrigin);
2672 // No point in adding something that might result in 0 origin value.
2673 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2674 Value *Cond = MSV->convertToBool(V: OpShadow, IRB);
2675 Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2676 }
2677 }
2678 }
2679 return *this;
2680 }
2681
2682 /// Add an application value to the mix.
2683 Combiner &Add(Value *V) {
2684 Value *OpShadow = MSV->getShadow(V);
2685 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2686 return Add(OpShadow, OpOrigin);
2687 }
2688
2689 /// Set the current combined values as the given instruction's shadow
2690 /// and origin.
2691 void Done(Instruction *I) {
2692 if (CombineShadow) {
2693 assert(Shadow);
2694 Shadow = MSV->CreateShadowCast(IRB, V: Shadow, dstTy: MSV->getShadowTy(V: I));
2695 MSV->setShadow(V: I, SV: Shadow);
2696 }
2697 if (MSV->MS.TrackOrigins) {
2698 assert(Origin);
2699 MSV->setOrigin(V: I, Origin);
2700 }
2701 }
2702
2703 /// Store the current combined value at the specified origin
2704 /// location.
2705 void DoneAndStoreOrigin(TypeSize TS, Value *OriginPtr) {
2706 if (MSV->MS.TrackOrigins) {
2707 assert(Origin);
2708 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, Alignment: kMinOriginAlignment);
2709 }
2710 }
2711 };
2712
2713 using ShadowAndOriginCombiner = Combiner<true>;
2714 using OriginCombiner = Combiner<false>;
2715
2716 /// Propagate origin for arbitrary operation.
2717 void setOriginForNaryOp(Instruction &I) {
2718 if (!MS.TrackOrigins)
2719 return;
2720 IRBuilder<> IRB(&I);
2721 OriginCombiner OC(this, IRB);
2722 for (Use &Op : I.operands())
2723 OC.Add(V: Op.get());
2724 OC.Done(I: &I);
2725 }
2726
2727 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2728 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2729 "Vector of pointers is not a valid shadow type");
2730 return Ty->isVectorTy() ? cast<FixedVectorType>(Val: Ty)->getNumElements() *
2731 Ty->getScalarSizeInBits()
2732 : Ty->getPrimitiveSizeInBits();
2733 }
2734
2735 /// Cast between two shadow types, extending or truncating as
2736 /// necessary.
2737 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2738 bool Signed = false) {
2739 Type *srcTy = V->getType();
2740 if (srcTy == dstTy)
2741 return V;
2742 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: srcTy);
2743 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: dstTy);
2744 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2745 return IRB.CreateICmpNE(LHS: V, RHS: getCleanShadow(V));
2746
2747 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2748 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2749 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2750 cast<VectorType>(Val: dstTy)->getElementCount() ==
2751 cast<VectorType>(Val: srcTy)->getElementCount())
2752 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2753 Value *V1 = IRB.CreateBitCast(V, DestTy: Type::getIntNTy(C&: *MS.C, N: srcSizeInBits));
2754 Value *V2 =
2755 IRB.CreateIntCast(V: V1, DestTy: Type::getIntNTy(C&: *MS.C, N: dstSizeInBits), isSigned: Signed);
2756 return IRB.CreateBitCast(V: V2, DestTy: dstTy);
2757 // TODO: handle struct types.
2758 }
2759
2760 /// Cast an application value to the type of its own shadow.
2761 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2762 Type *ShadowTy = getShadowTy(V);
2763 if (V->getType() == ShadowTy)
2764 return V;
2765 if (V->getType()->isPtrOrPtrVectorTy())
2766 return IRB.CreatePtrToInt(V, DestTy: ShadowTy);
2767 else
2768 return IRB.CreateBitCast(V, DestTy: ShadowTy);
2769 }
2770
2771 /// Propagate shadow for arbitrary operation.
2772 void handleShadowOr(Instruction &I) {
2773 IRBuilder<> IRB(&I);
2774 ShadowAndOriginCombiner SC(this, IRB);
2775 for (Use &Op : I.operands())
2776 SC.Add(V: Op.get());
2777 SC.Done(I: &I);
2778 }
2779
2780 // Perform a bitwise OR on the horizontal pairs (or other specified grouping)
2781 // of elements.
2782 //
2783 // For example, suppose we have:
2784 // VectorA: <a0, a1, a2, a3, a4, a5>
2785 // VectorB: <b0, b1, b2, b3, b4, b5>
2786 // ReductionFactor: 3
2787 // Shards: 1
2788 // The output would be:
2789 // <a0|a1|a2, a3|a4|a5, b0|b1|b2, b3|b4|b5>
2790 //
2791 // If we have:
2792 // VectorA: <a0, a1, a2, a3, a4, a5, a6, a7>
2793 // VectorB: <b0, b1, b2, b3, b4, b5, b6, b7>
2794 // ReductionFactor: 2
2795 // Shards: 2
2796 // then a and be each have 2 "shards", resulting in the output being
2797 // interleaved:
2798 // <a0|a1, a2|a3, b0|b1, b2|b3, a4|a5, a6|a7, b4|b5, b6|b7>
2799 //
2800 // This is convenient for instrumenting horizontal add/sub.
2801 // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic().
2802 Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor,
2803 unsigned Shards, Value *VectorA, Value *VectorB) {
2804 assert(isa<FixedVectorType>(VectorA->getType()));
2805 unsigned NumElems =
2806 cast<FixedVectorType>(Val: VectorA->getType())->getNumElements();
2807
2808 [[maybe_unused]] unsigned TotalNumElems = NumElems;
2809 if (VectorB) {
2810 assert(VectorA->getType() == VectorB->getType());
2811 TotalNumElems *= 2;
2812 }
2813
2814 assert(NumElems % (ReductionFactor * Shards) == 0);
2815
2816 Value *Or = nullptr;
2817
2818 IRBuilder<> IRB(&I);
2819 for (unsigned i = 0; i < ReductionFactor; i++) {
2820 SmallVector<int, 16> Mask;
2821
2822 for (unsigned j = 0; j < Shards; j++) {
2823 unsigned Offset = NumElems / Shards * j;
2824
2825 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2826 Mask.push_back(Elt: Offset + X + i);
2827
2828 if (VectorB) {
2829 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2830 Mask.push_back(Elt: NumElems + Offset + X + i);
2831 }
2832 }
2833
2834 Value *Masked;
2835 if (VectorB)
2836 Masked = IRB.CreateShuffleVector(V1: VectorA, V2: VectorB, Mask);
2837 else
2838 Masked = IRB.CreateShuffleVector(V: VectorA, Mask);
2839
2840 if (Or)
2841 Or = IRB.CreateOr(LHS: Or, RHS: Masked);
2842 else
2843 Or = Masked;
2844 }
2845
2846 return Or;
2847 }
2848
2849 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
2850 /// fields.
2851 ///
2852 /// e.g., <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16>)
2853 /// <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8>, <16 x i8>)
2854 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards) {
2855 assert(I.arg_size() == 1 || I.arg_size() == 2);
2856
2857 assert(I.getType()->isVectorTy());
2858 assert(I.getArgOperand(0)->getType()->isVectorTy());
2859
2860 [[maybe_unused]] FixedVectorType *ParamType =
2861 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
2862 assert((I.arg_size() != 2) ||
2863 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
2864 [[maybe_unused]] FixedVectorType *ReturnType =
2865 cast<FixedVectorType>(Val: I.getType());
2866 assert(ParamType->getNumElements() * I.arg_size() ==
2867 2 * ReturnType->getNumElements());
2868
2869 IRBuilder<> IRB(&I);
2870
2871 // Horizontal OR of shadow
2872 Value *FirstArgShadow = getShadow(I: &I, i: 0);
2873 Value *SecondArgShadow = nullptr;
2874 if (I.arg_size() == 2)
2875 SecondArgShadow = getShadow(I: &I, i: 1);
2876
2877 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
2878 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
2879
2880 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
2881
2882 setShadow(V: &I, SV: OrShadow);
2883 setOriginForNaryOp(I);
2884 }
2885
2886 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
2887 /// fields, with the parameters reinterpreted to have elements of a specified
2888 /// width. For example:
2889 /// @llvm.x86.ssse3.phadd.w(<1 x i64> [[VAR1]], <1 x i64> [[VAR2]])
2890 /// conceptually operates on
2891 /// (<4 x i16> [[VAR1]], <4 x i16> [[VAR2]])
2892 /// and can be handled with ReinterpretElemWidth == 16.
2893 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards,
2894 int ReinterpretElemWidth) {
2895 assert(I.arg_size() == 1 || I.arg_size() == 2);
2896
2897 assert(I.getType()->isVectorTy());
2898 assert(I.getArgOperand(0)->getType()->isVectorTy());
2899
2900 FixedVectorType *ParamType =
2901 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
2902 assert((I.arg_size() != 2) ||
2903 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
2904
2905 [[maybe_unused]] FixedVectorType *ReturnType =
2906 cast<FixedVectorType>(Val: I.getType());
2907 assert(ParamType->getNumElements() * I.arg_size() ==
2908 2 * ReturnType->getNumElements());
2909
2910 IRBuilder<> IRB(&I);
2911
2912 FixedVectorType *ReinterpretShadowTy = nullptr;
2913 assert(isAligned(Align(ReinterpretElemWidth),
2914 ParamType->getPrimitiveSizeInBits()));
2915 ReinterpretShadowTy = FixedVectorType::get(
2916 ElementType: IRB.getIntNTy(N: ReinterpretElemWidth),
2917 NumElts: ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth);
2918
2919 // Horizontal OR of shadow
2920 Value *FirstArgShadow = getShadow(I: &I, i: 0);
2921 FirstArgShadow = IRB.CreateBitCast(V: FirstArgShadow, DestTy: ReinterpretShadowTy);
2922
2923 // If we had two parameters each with an odd number of elements, the total
2924 // number of elements is even, but we have never seen this in extant
2925 // instruction sets, so we enforce that each parameter must have an even
2926 // number of elements.
2927 assert(isAligned(
2928 Align(2),
2929 cast<FixedVectorType>(FirstArgShadow->getType())->getNumElements()));
2930
2931 Value *SecondArgShadow = nullptr;
2932 if (I.arg_size() == 2) {
2933 SecondArgShadow = getShadow(I: &I, i: 1);
2934 SecondArgShadow = IRB.CreateBitCast(V: SecondArgShadow, DestTy: ReinterpretShadowTy);
2935 }
2936
2937 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
2938 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
2939
2940 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
2941
2942 setShadow(V: &I, SV: OrShadow);
2943 setOriginForNaryOp(I);
2944 }
2945
2946 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2947
2948 // Handle multiplication by constant.
2949 //
2950 // Handle a special case of multiplication by constant that may have one or
2951 // more zeros in the lower bits. This makes corresponding number of lower bits
2952 // of the result zero as well. We model it by shifting the other operand
2953 // shadow left by the required number of bits. Effectively, we transform
2954 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2955 // We use multiplication by 2**N instead of shift to cover the case of
2956 // multiplication by 0, which may occur in some elements of a vector operand.
2957 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2958 Value *OtherArg) {
2959 Constant *ShadowMul;
2960 Type *Ty = ConstArg->getType();
2961 if (auto *VTy = dyn_cast<VectorType>(Val: Ty)) {
2962 unsigned NumElements = cast<FixedVectorType>(Val: VTy)->getNumElements();
2963 Type *EltTy = VTy->getElementType();
2964 SmallVector<Constant *, 16> Elements;
2965 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2966 if (ConstantInt *Elt =
2967 dyn_cast<ConstantInt>(Val: ConstArg->getAggregateElement(Elt: Idx))) {
2968 const APInt &V = Elt->getValue();
2969 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2970 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: V2));
2971 } else {
2972 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: 1));
2973 }
2974 }
2975 ShadowMul = ConstantVector::get(V: Elements);
2976 } else {
2977 if (ConstantInt *Elt = dyn_cast<ConstantInt>(Val: ConstArg)) {
2978 const APInt &V = Elt->getValue();
2979 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2980 ShadowMul = ConstantInt::get(Ty, V: V2);
2981 } else {
2982 ShadowMul = ConstantInt::get(Ty, V: 1);
2983 }
2984 }
2985
2986 IRBuilder<> IRB(&I);
2987 setShadow(V: &I,
2988 SV: IRB.CreateMul(LHS: getShadow(V: OtherArg), RHS: ShadowMul, Name: "msprop_mul_cst"));
2989 setOrigin(V: &I, Origin: getOrigin(V: OtherArg));
2990 }
2991
2992 void visitMul(BinaryOperator &I) {
2993 Constant *constOp0 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0));
2994 Constant *constOp1 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1));
2995 if (constOp0 && !constOp1)
2996 handleMulByConstant(I, ConstArg: constOp0, OtherArg: I.getOperand(i_nocapture: 1));
2997 else if (constOp1 && !constOp0)
2998 handleMulByConstant(I, ConstArg: constOp1, OtherArg: I.getOperand(i_nocapture: 0));
2999 else
3000 handleShadowOr(I);
3001 }
3002
3003 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
3004 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
3005 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
3006 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
3007 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
3008 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
3009
3010 void handleIntegerDiv(Instruction &I) {
3011 IRBuilder<> IRB(&I);
3012 // Strict on the second argument.
3013 insertCheckShadowOf(Val: I.getOperand(i: 1), OrigIns: &I);
3014 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3015 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3016 }
3017
3018 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3019 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3020 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
3021 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
3022
3023 // Floating point division is side-effect free. We can not require that the
3024 // divisor is fully initialized and must propagate shadow. See PR37523.
3025 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
3026 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
3027
3028 /// Instrument == and != comparisons.
3029 ///
3030 /// Sometimes the comparison result is known even if some of the bits of the
3031 /// arguments are not.
3032 void handleEqualityComparison(ICmpInst &I) {
3033 IRBuilder<> IRB(&I);
3034 Value *A = I.getOperand(i_nocapture: 0);
3035 Value *B = I.getOperand(i_nocapture: 1);
3036 Value *Sa = getShadow(V: A);
3037 Value *Sb = getShadow(V: B);
3038
3039 Value *Si = propagateEqualityComparison(IRB, A, B, Sa, Sb);
3040
3041 setShadow(V: &I, SV: Si);
3042 setOriginForNaryOp(I);
3043 }
3044
3045 /// Instrument relational comparisons.
3046 ///
3047 /// This function does exact shadow propagation for all relational
3048 /// comparisons of integers, pointers and vectors of those.
3049 /// FIXME: output seems suboptimal when one of the operands is a constant
3050 void handleRelationalComparisonExact(ICmpInst &I) {
3051 IRBuilder<> IRB(&I);
3052 Value *A = I.getOperand(i_nocapture: 0);
3053 Value *B = I.getOperand(i_nocapture: 1);
3054 Value *Sa = getShadow(V: A);
3055 Value *Sb = getShadow(V: B);
3056
3057 // Get rid of pointers and vectors of pointers.
3058 // For ints (and vectors of ints), types of A and Sa match,
3059 // and this is a no-op.
3060 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
3061 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
3062
3063 // Let [a0, a1] be the interval of possible values of A, taking into account
3064 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
3065 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
3066 bool IsSigned = I.isSigned();
3067
3068 auto GetMinMaxUnsigned = [&](Value *V, Value *S) {
3069 if (IsSigned) {
3070 // Sign-flip to map from signed range to unsigned range. Relation A vs B
3071 // should be preserved, if checked with `getUnsignedPredicate()`.
3072 // Relationship between Amin, Amax, Bmin, Bmax also will not be
3073 // affected, as they are created by effectively adding/substructing from
3074 // A (or B) a value, derived from shadow, with no overflow, either
3075 // before or after sign flip.
3076 APInt MinVal =
3077 APInt::getSignedMinValue(numBits: V->getType()->getScalarSizeInBits());
3078 V = IRB.CreateXor(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: MinVal));
3079 }
3080 // Minimize undefined bits.
3081 Value *Min = IRB.CreateAnd(LHS: V, RHS: IRB.CreateNot(V: S));
3082 Value *Max = IRB.CreateOr(LHS: V, RHS: S);
3083 return std::make_pair(x&: Min, y&: Max);
3084 };
3085
3086 auto [Amin, Amax] = GetMinMaxUnsigned(A, Sa);
3087 auto [Bmin, Bmax] = GetMinMaxUnsigned(B, Sb);
3088 Value *S1 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amin, RHS: Bmax);
3089 Value *S2 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amax, RHS: Bmin);
3090
3091 Value *Si = IRB.CreateXor(LHS: S1, RHS: S2);
3092 setShadow(V: &I, SV: Si);
3093 setOriginForNaryOp(I);
3094 }
3095
3096 /// Instrument signed relational comparisons.
3097 ///
3098 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
3099 /// bit of the shadow. Everything else is delegated to handleShadowOr().
3100 void handleSignedRelationalComparison(ICmpInst &I) {
3101 Constant *constOp;
3102 Value *op = nullptr;
3103 CmpInst::Predicate pre;
3104 if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3105 op = I.getOperand(i_nocapture: 0);
3106 pre = I.getPredicate();
3107 } else if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0)))) {
3108 op = I.getOperand(i_nocapture: 1);
3109 pre = I.getSwappedPredicate();
3110 } else {
3111 handleShadowOr(I);
3112 return;
3113 }
3114
3115 if ((constOp->isNullValue() &&
3116 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
3117 (constOp->isAllOnesValue() &&
3118 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
3119 IRBuilder<> IRB(&I);
3120 Value *Shadow = IRB.CreateICmpSLT(LHS: getShadow(V: op), RHS: getCleanShadow(V: op),
3121 Name: "_msprop_icmp_s");
3122 setShadow(V: &I, SV: Shadow);
3123 setOrigin(V: &I, Origin: getOrigin(V: op));
3124 } else {
3125 handleShadowOr(I);
3126 }
3127 }
3128
3129 void visitICmpInst(ICmpInst &I) {
3130 if (!Opts.msan_handle_icmp) {
3131 handleShadowOr(I);
3132 return;
3133 }
3134 if (I.isEquality()) {
3135 handleEqualityComparison(I);
3136 return;
3137 }
3138
3139 assert(I.isRelational());
3140 if (Opts.msan_handle_icmp_exact) {
3141 handleRelationalComparisonExact(I);
3142 return;
3143 }
3144 if (I.isSigned()) {
3145 handleSignedRelationalComparison(I);
3146 return;
3147 }
3148
3149 assert(I.isUnsigned());
3150 if ((isa<Constant>(Val: I.getOperand(i_nocapture: 0)) || isa<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3151 handleRelationalComparisonExact(I);
3152 return;
3153 }
3154
3155 handleShadowOr(I);
3156 }
3157
3158 void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
3159
3160 void handleShift(BinaryOperator &I) {
3161 IRBuilder<> IRB(&I);
3162 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3163 // Otherwise perform the same shift on S1.
3164 Value *S1 = getShadow(I: &I, i: 0);
3165 Value *S2 = getShadow(I: &I, i: 1);
3166 Value *S2Conv =
3167 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3168 Value *V2 = I.getOperand(i_nocapture: 1);
3169 Value *Shift = IRB.CreateBinOp(Opc: I.getOpcode(), LHS: S1, RHS: V2);
3170 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3171 setOriginForNaryOp(I);
3172 }
3173
3174 void visitShl(BinaryOperator &I) { handleShift(I); }
3175 void visitAShr(BinaryOperator &I) { handleShift(I); }
3176 void visitLShr(BinaryOperator &I) { handleShift(I); }
3177
3178 void handleFunnelShift(IntrinsicInst &I) {
3179 IRBuilder<> IRB(&I);
3180 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3181 // Otherwise perform the same shift on S0 and S1.
3182 Value *S0 = getShadow(I: &I, i: 0);
3183 Value *S1 = getShadow(I: &I, i: 1);
3184 Value *S2 = getShadow(I: &I, i: 2);
3185 Value *S2Conv =
3186 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3187 Value *V2 = I.getOperand(i_nocapture: 2);
3188 Value *Shift = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: S2Conv->getType(),
3189 Args: {S0, S1, V2});
3190 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3191 setOriginForNaryOp(I);
3192 }
3193
3194 // Instrument bit manipulation intrinsics.
3195 // All of these intrinsics are Z = I(SRC, MASK)
3196 // where the types of all operands and the result match.
3197 // The following instrumentation happens to work for all of them:
3198 // Sz = I(Ssrc, MASK) | (sext (Smask != 0))
3199 void handleGenericBitManipulation(IntrinsicInst &I) {
3200 IRBuilder<> IRB(&I);
3201 Type *ShadowTy = getShadowTy(V: &I);
3202
3203 // If any bit of the mask operand is poisoned, then the whole thing is.
3204 Value *SMask = getShadow(I: &I, i: 1);
3205 SMask = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SMask, RHS: getCleanShadow(OrigTy: ShadowTy)),
3206 DestTy: ShadowTy);
3207 // Apply the same intrinsic to the shadow of the first operand.
3208 Value *S;
3209 if (Function *Func = I.getCalledFunction())
3210 S = IRB.CreateCall(Callee: Func, Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3211 else
3212 S = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: ShadowTy,
3213 Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3214
3215 setShadow(V: &I, SV: IRB.CreateOr(LHS: SMask, RHS: S));
3216 setOriginForNaryOp(I);
3217 }
3218
3219 /// Instrument llvm.memmove
3220 ///
3221 /// At this point we don't know if llvm.memmove will be inlined or not.
3222 /// If we don't instrument it and it gets inlined,
3223 /// our interceptor will not kick in and we will lose the memmove.
3224 /// If we instrument the call here, but it does not get inlined,
3225 /// we will memmove the shadow twice: which is bad in case
3226 /// of overlapping regions. So, we simply lower the intrinsic to a call.
3227 ///
3228 /// Similar situation exists for memcpy and memset.
3229 void visitMemMoveInst(MemMoveInst &I) {
3230 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3231 IRBuilder<> IRB(&I);
3232 IRB.CreateCall(Callee: MS.MemmoveFn,
3233 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3234 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3235 I.eraseFromParent();
3236 }
3237
3238 /// Instrument memcpy
3239 ///
3240 /// Similar to memmove: avoid copying shadow twice. This is somewhat
3241 /// unfortunate as it may slowdown small constant memcpys.
3242 /// FIXME: consider doing manual inline for small constant sizes and proper
3243 /// alignment.
3244 ///
3245 /// Note: This also handles memcpy.inline, which promises no calls to external
3246 /// functions as an optimization. However, with instrumentation enabled this
3247 /// is difficult to promise; additionally, we know that the MSan runtime
3248 /// exists and provides __msan_memcpy(). Therefore, we assume that with
3249 /// instrumentation it's safe to turn memcpy.inline into a call to
3250 /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
3251 /// itself, instrumentation should be disabled with the no_sanitize attribute.
3252 void visitMemCpyInst(MemCpyInst &I) {
3253 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3254 IRBuilder<> IRB(&I);
3255 IRB.CreateCall(Callee: MS.MemcpyFn,
3256 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3257 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3258 I.eraseFromParent();
3259 }
3260
3261 // Same as memcpy.
3262 void visitMemSetInst(MemSetInst &I) {
3263 IRBuilder<> IRB(&I);
3264 IRB.CreateCall(
3265 Callee: MS.MemsetFn,
3266 Args: {I.getArgOperand(i: 0),
3267 IRB.CreateIntCast(V: I.getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
3268 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3269 I.eraseFromParent();
3270 }
3271
3272 void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
3273
3274 void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
3275
3276 /// Handle vector store-like intrinsics.
3277 ///
3278 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
3279 /// has 1 pointer argument and 1 vector argument, returns void.
3280 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
3281 assert(I.arg_size() == 2);
3282
3283 IRBuilder<> IRB(&I);
3284 Value *Addr = I.getArgOperand(i: 0);
3285 Value *Shadow = getShadow(I: &I, i: 1);
3286 Value *ShadowPtr, *OriginPtr;
3287
3288 // We don't know the pointer alignment (could be unaligned SSE store!).
3289 // Have to assume to worst case.
3290 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
3291 Addr, IRB, ShadowTy: Shadow->getType(), Alignment: Align(1), /*isStore*/ true);
3292 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1));
3293
3294 if (Opts.msan_check_access_address)
3295 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3296
3297 // FIXME: factor out common code from materializeStores
3298 if (MS.TrackOrigins)
3299 IRB.CreateStore(Val: getOrigin(I: &I, i: 1), Ptr: OriginPtr);
3300 return true;
3301 }
3302
3303 /// Handle vector load-like intrinsics.
3304 ///
3305 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
3306 /// has 1 pointer argument, returns a vector.
3307 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
3308 assert(I.arg_size() == 1);
3309
3310 IRBuilder<> IRB(&I);
3311 Value *Addr = I.getArgOperand(i: 0);
3312
3313 Type *ShadowTy = getShadowTy(V: &I);
3314 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
3315 if (PropagateShadow) {
3316 // We don't know the pointer alignment (could be unaligned SSE load!).
3317 // Have to assume to worst case.
3318 const Align Alignment = Align(1);
3319 std::tie(args&: ShadowPtr, args&: OriginPtr) =
3320 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3321 setShadow(V: &I,
3322 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
3323 } else {
3324 setShadow(V: &I, SV: getCleanShadow(V: &I));
3325 }
3326
3327 if (Opts.msan_check_access_address)
3328 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3329
3330 if (MS.TrackOrigins) {
3331 if (PropagateShadow)
3332 setOrigin(V: &I, Origin: IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
3333 else
3334 setOrigin(V: &I, Origin: getCleanOrigin());
3335 }
3336 return true;
3337 }
3338
3339 /// Handle (SIMD arithmetic)-like intrinsics.
3340 ///
3341 /// Instrument intrinsics with any number of arguments of the same type [*],
3342 /// equal to the return type, plus a specified number of trailing flags of
3343 /// any type.
3344 ///
3345 /// [*] The type should be simple (no aggregates or pointers; vectors are
3346 /// fine).
3347 ///
3348 /// Caller guarantees that this intrinsic does not access memory.
3349 ///
3350 /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
3351 /// by this handler. See horizontalReduce().
3352 ///
3353 /// TODO: permutation intrinsics are also often incorrectly matched.
3354 [[maybe_unused]] bool
3355 maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
3356 unsigned int trailingFlags) {
3357 Type *RetTy = I.getType();
3358 if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy()))
3359 return false;
3360
3361 unsigned NumArgOperands = I.arg_size();
3362 assert(NumArgOperands >= trailingFlags);
3363 for (unsigned i = 0; i < NumArgOperands - trailingFlags; ++i) {
3364 Type *Ty = I.getArgOperand(i)->getType();
3365 if (Ty != RetTy)
3366 return false;
3367 }
3368
3369 IRBuilder<> IRB(&I);
3370 ShadowAndOriginCombiner SC(this, IRB);
3371 for (unsigned i = 0; i < NumArgOperands; ++i)
3372 SC.Add(V: I.getArgOperand(i));
3373 SC.Done(I: &I);
3374
3375 return true;
3376 }
3377
3378 /// Returns whether it was able to heuristically instrument unknown
3379 /// intrinsics.
3380 ///
3381 /// The main purpose of this code is to do something reasonable with all
3382 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
3383 /// We recognize several classes of intrinsics by their argument types and
3384 /// ModRefBehaviour and apply special instrumentation when we are reasonably
3385 /// sure that we know what the intrinsic does.
3386 ///
3387 /// We special-case intrinsics where this approach fails. See llvm.bswap
3388 /// handling as an example of that.
3389 bool maybeHandleUnknownIntrinsicUnlogged(IntrinsicInst &I) {
3390 unsigned NumArgOperands = I.arg_size();
3391 if (NumArgOperands == 0)
3392 return false;
3393
3394 if (NumArgOperands == 2 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3395 I.getArgOperand(i: 1)->getType()->isVectorTy() &&
3396 I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
3397 // This looks like a vector store.
3398 return handleVectorStoreIntrinsic(I);
3399 }
3400
3401 if (NumArgOperands == 1 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3402 I.getType()->isVectorTy() && I.onlyReadsMemory()) {
3403 // This looks like a vector load.
3404 return handleVectorLoadIntrinsic(I);
3405 }
3406
3407 if (I.doesNotAccessMemory())
3408 if (maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/0))
3409 return true;
3410
3411 // FIXME: detect and handle SSE maskstore/maskload?
3412 // Some cases are now handled in handleAVXMasked{Load,Store}.
3413 return false;
3414 }
3415
3416 bool maybeHandleUnknownIntrinsic(IntrinsicInst &I) {
3417 if (maybeHandleUnknownIntrinsicUnlogged(I)) {
3418 if (Opts.msan_dump_heuristic_instructions)
3419 dumpInst(I, Prefix: "Heuristic");
3420
3421 LLVM_DEBUG(dbgs() << "UNKNOWN INSTRUCTION HANDLED HEURISTICALLY: " << I
3422 << "\n");
3423 return true;
3424 } else
3425 return false;
3426 }
3427
3428 void handleInvariantGroup(IntrinsicInst &I) {
3429 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3430 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3431 }
3432
3433 void handleLifetimeStart(IntrinsicInst &I) {
3434 if (!PoisonStack)
3435 return;
3436 AllocaInst *AI = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
3437 if (AI)
3438 LifetimeStartList.push_back(Elt: std::make_pair(x: &I, y&: AI));
3439 }
3440
3441 void handleBswap(IntrinsicInst &I) {
3442 IRBuilder<> IRB(&I);
3443 Value *Op = I.getArgOperand(i: 0);
3444 Type *OpType = Op->getType();
3445 setShadow(V: &I, SV: IRB.CreateIntrinsic(ID: Intrinsic::bswap, OverloadTypes: ArrayRef(&OpType, 1),
3446 Args: getShadow(V: Op)));
3447 setOrigin(V: &I, Origin: getOrigin(V: Op));
3448 }
3449
3450 // Uninitialized bits are ok if they appear after the leading/trailing 0's
3451 // and a 1. If the input is all zero, it is fully initialized iff
3452 // !is_zero_poison.
3453 //
3454 // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
3455 // concrete value 0/1, and ? is an uninitialized bit:
3456 // - 0001 0??? is fully initialized
3457 // - 000? ???? is fully uninitialized (*)
3458 // - ???? ???? is fully uninitialized
3459 // - 0000 0000 is fully uninitialized if is_zero_poison,
3460 // fully initialized otherwise
3461 //
3462 // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
3463 // only need to poison 4 bits.
3464 //
3465 // OutputShadow =
3466 // ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
3467 // || (is_zero_poison && AllZeroSrc)
3468 void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
3469 IRBuilder<> IRB(&I);
3470 Value *Src = I.getArgOperand(i: 0);
3471 Value *SrcShadow = getShadow(V: Src);
3472
3473 Value *False = IRB.getInt1(V: false);
3474 Value *ConcreteZerosCount = IRB.CreateIntrinsic(
3475 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Src, /*is_zero_poison=*/False});
3476 Value *ShadowZerosCount = IRB.CreateIntrinsic(
3477 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {SrcShadow, /*is_zero_poison=*/False});
3478
3479 Value *CompareConcreteZeros = IRB.CreateICmpUGE(
3480 LHS: ConcreteZerosCount, RHS: ShadowZerosCount, Name: "_mscz_cmp_zeros");
3481
3482 Value *NotAllZeroShadow =
3483 IRB.CreateIsNotNull(Arg: SrcShadow, Name: "_mscz_shadow_not_null");
3484 Value *OutputShadow =
3485 IRB.CreateAnd(LHS: CompareConcreteZeros, RHS: NotAllZeroShadow, Name: "_mscz_main");
3486
3487 // If zero poison is requested, mix in with the shadow
3488 Constant *IsZeroPoison = cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3489 if (!IsZeroPoison->isNullValue()) {
3490 Value *BoolZeroPoison = IRB.CreateIsNull(Arg: Src, Name: "_mscz_bzp");
3491 OutputShadow = IRB.CreateOr(LHS: OutputShadow, RHS: BoolZeroPoison, Name: "_mscz_bs");
3492 }
3493
3494 OutputShadow = IRB.CreateSExt(V: OutputShadow, DestTy: getShadowTy(V: Src), Name: "_mscz_os");
3495
3496 setShadow(V: &I, SV: OutputShadow);
3497 setOriginForNaryOp(I);
3498 }
3499
3500 /// Some instructions have additional zero-elements in the return type
3501 /// e.g., <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, ...)
3502 ///
3503 /// This function will return a vector type with the same number of elements
3504 /// as the input, but same per-element width as the return value e.g.,
3505 /// <8 x i8>.
3506 FixedVectorType *maybeShrinkVectorShadowType(Value *Src, IntrinsicInst &I) {
3507 assert(isa<FixedVectorType>(getShadowTy(&I)));
3508 FixedVectorType *ShadowType = cast<FixedVectorType>(Val: getShadowTy(V: &I));
3509
3510 // TODO: generalize beyond 2x?
3511 if (ShadowType->getElementCount() ==
3512 cast<VectorType>(Val: Src->getType())->getElementCount() * 2)
3513 ShadowType = FixedVectorType::getHalfElementsVectorType(VTy: ShadowType);
3514
3515 assert(ShadowType->getElementCount() ==
3516 cast<VectorType>(Src->getType())->getElementCount());
3517
3518 return ShadowType;
3519 }
3520
3521 /// Doubles the length of a vector shadow (extending with zeros) if necessary
3522 /// to match the length of the shadow for the instruction.
3523 /// If scalar types of the vectors are different, it will use the type of the
3524 /// input vector.
3525 /// This is more type-safe than CreateShadowCast().
3526 Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
3527 IRBuilder<> IRB(&I);
3528 assert(isa<FixedVectorType>(Shadow->getType()));
3529 assert(isa<FixedVectorType>(I.getType()));
3530
3531 Value *FullShadow = getCleanShadow(V: &I);
3532 unsigned ShadowNumElems =
3533 cast<FixedVectorType>(Val: Shadow->getType())->getNumElements();
3534 unsigned FullShadowNumElems =
3535 cast<FixedVectorType>(Val: FullShadow->getType())->getNumElements();
3536
3537 assert((ShadowNumElems == FullShadowNumElems) ||
3538 (ShadowNumElems * 2 == FullShadowNumElems));
3539
3540 if (ShadowNumElems == FullShadowNumElems) {
3541 FullShadow = Shadow;
3542 } else {
3543 // TODO: generalize beyond 2x?
3544 SmallVector<int, 32> ShadowMask(FullShadowNumElems);
3545 std::iota(first: ShadowMask.begin(), last: ShadowMask.end(), value: 0);
3546
3547 // Append zeros
3548 FullShadow =
3549 IRB.CreateShuffleVector(V1: Shadow, V2: getCleanShadow(V: Shadow), Mask: ShadowMask);
3550 }
3551
3552 return FullShadow;
3553 }
3554
3555 /// Handle x86 SSE vector conversion.
3556 ///
3557 /// e.g., single-precision to half-precision conversion:
3558 /// <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
3559 /// <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0)
3560 ///
3561 /// floating-point to integer:
3562 /// <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>)
3563 /// <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>)
3564 ///
3565 /// Note: if the output has more elements, they are zero-initialized (and
3566 /// therefore the shadow will also be initialized).
3567 ///
3568 /// This differs from handleSSEVectorConvertIntrinsic() because it
3569 /// propagates uninitialized shadow (instead of checking the shadow).
3570 void handleSSEVectorConvertIntrinsicByProp(IntrinsicInst &I,
3571 bool HasRoundingMode) {
3572 if (HasRoundingMode) {
3573 assert(I.arg_size() == 2);
3574 [[maybe_unused]] Value *RoundingMode = I.getArgOperand(i: 1);
3575 assert(RoundingMode->getType()->isIntegerTy());
3576 } else {
3577 assert(I.arg_size() == 1);
3578 }
3579
3580 Value *Src = I.getArgOperand(i: 0);
3581 assert(Src->getType()->isVectorTy());
3582
3583 // The return type might have more elements than the input.
3584 // Temporarily shrink the return type's number of elements.
3585 VectorType *ShadowType = maybeShrinkVectorShadowType(Src, I);
3586
3587 IRBuilder<> IRB(&I);
3588 Value *S0 = getShadow(I: &I, i: 0);
3589
3590 /// For scalars:
3591 /// Since they are converting to and/or from floating-point, the output is:
3592 /// - fully uninitialized if *any* bit of the input is uninitialized
3593 /// - fully ininitialized if all bits of the input are ininitialized
3594 /// We apply the same principle on a per-field basis for vectors.
3595 Value *Shadow =
3596 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)), DestTy: ShadowType);
3597
3598 // The return type might have more elements than the input.
3599 // Extend the return type back to its original width if necessary.
3600 Value *FullShadow = maybeExtendVectorShadowWithZeros(Shadow, I);
3601
3602 setShadow(V: &I, SV: FullShadow);
3603 setOriginForNaryOp(I);
3604 }
3605
3606 // Instrument x86 SSE vector convert intrinsic.
3607 //
3608 // This function instruments intrinsics like cvtsi2ss:
3609 // %Out = int_xxx_cvtyyy(%ConvertOp)
3610 // or
3611 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
3612 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
3613 // number \p Out elements, and (if has 2 arguments) copies the rest of the
3614 // elements from \p CopyOp.
3615 // In most cases conversion involves floating-point value which may trigger a
3616 // hardware exception when not fully initialized. For this reason we require
3617 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
3618 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
3619 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
3620 // return a fully initialized value.
3621 //
3622 // For Arm NEON vector convert intrinsics, see
3623 // handleNEONVectorConvertIntrinsic().
3624 void handleSSEVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
3625 bool HasRoundingMode = false) {
3626 IRBuilder<> IRB(&I);
3627 Value *CopyOp, *ConvertOp;
3628
3629 assert((!HasRoundingMode ||
3630 isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
3631 "Invalid rounding mode");
3632
3633 switch (I.arg_size() - HasRoundingMode) {
3634 case 2:
3635 CopyOp = I.getArgOperand(i: 0);
3636 ConvertOp = I.getArgOperand(i: 1);
3637 break;
3638 case 1:
3639 ConvertOp = I.getArgOperand(i: 0);
3640 CopyOp = nullptr;
3641 break;
3642 default:
3643 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
3644 }
3645
3646 // The first *NumUsedElements* elements of ConvertOp are converted to the
3647 // same number of output elements. The rest of the output is copied from
3648 // CopyOp, or (if not available) filled with zeroes.
3649 // Combine shadow for elements of ConvertOp that are used in this operation,
3650 // and insert a check.
3651 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
3652 // int->any conversion.
3653 Value *ConvertShadow = getShadow(V: ConvertOp);
3654 Value *AggShadow = nullptr;
3655 if (ConvertOp->getType()->isVectorTy()) {
3656 AggShadow = IRB.CreateExtractElement(
3657 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
3658 for (int i = 1; i < NumUsedElements; ++i) {
3659 Value *MoreShadow = IRB.CreateExtractElement(
3660 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3661 AggShadow = IRB.CreateOr(LHS: AggShadow, RHS: MoreShadow);
3662 }
3663 } else {
3664 AggShadow = ConvertShadow;
3665 }
3666 assert(AggShadow->getType()->isIntegerTy());
3667 insertCheckShadow(Shadow: AggShadow, Origin: getOrigin(V: ConvertOp), OrigIns: &I);
3668
3669 // Build result shadow by zero-filling parts of CopyOp shadow that come from
3670 // ConvertOp.
3671 if (CopyOp) {
3672 assert(CopyOp->getType() == I.getType());
3673 assert(CopyOp->getType()->isVectorTy());
3674 Value *ResultShadow = getShadow(V: CopyOp);
3675 Type *EltTy = cast<VectorType>(Val: ResultShadow->getType())->getElementType();
3676 for (int i = 0; i < NumUsedElements; ++i) {
3677 ResultShadow = IRB.CreateInsertElement(
3678 Vec: ResultShadow, NewElt: ConstantInt::getNullValue(Ty: EltTy),
3679 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3680 }
3681 setShadow(V: &I, SV: ResultShadow);
3682 setOrigin(V: &I, Origin: getOrigin(V: CopyOp));
3683 } else {
3684 setShadow(V: &I, SV: getCleanShadow(V: &I));
3685 setOrigin(V: &I, Origin: getCleanOrigin());
3686 }
3687 }
3688
3689 // Given a scalar or vector, extract lower 64 bits (or less), and return all
3690 // zeroes if it is zero, and all ones otherwise.
3691 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3692 if (S->getType()->isVectorTy())
3693 S = CreateShadowCast(IRB, V: S, dstTy: IRB.getInt64Ty(), /* Signed */ true);
3694 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
3695 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3696 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3697 }
3698
3699 // Given a vector, extract its first element, and return all
3700 // zeroes if it is zero, and all ones otherwise.
3701 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3702 Value *S1 = IRB.CreateExtractElement(Vec: S, Idx: (uint64_t)0);
3703 Value *S2 = IRB.CreateICmpNE(LHS: S1, RHS: getCleanShadow(V: S1));
3704 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3705 }
3706
3707 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3708 Type *T = S->getType();
3709 assert(T->isVectorTy());
3710 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3711 return IRB.CreateSExt(V: S2, DestTy: T);
3712 }
3713
3714 // Instrument vector shift intrinsic.
3715 //
3716 // This function instruments intrinsics like int_x86_avx2_psll_w.
3717 // Intrinsic shifts %In by %ShiftSize bits.
3718 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
3719 // size, and the rest is ignored. Behavior is defined even if shift size is
3720 // greater than register (or field) width.
3721 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3722 assert(I.arg_size() == 2);
3723 IRBuilder<> IRB(&I);
3724 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3725 // Otherwise perform the same shift on S1.
3726 Value *S1 = getShadow(I: &I, i: 0);
3727 Value *S2 = getShadow(I: &I, i: 1);
3728 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S: S2)
3729 : Lower64ShadowExtend(IRB, S: S2, T: getShadowTy(V: &I));
3730 Value *V1 = I.getOperand(i_nocapture: 0);
3731 Value *V2 = I.getOperand(i_nocapture: 1);
3732 Value *Shift = IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
3733 Args: {IRB.CreateBitCast(V: S1, DestTy: V1->getType()), V2});
3734 Shift = IRB.CreateBitCast(V: Shift, DestTy: getShadowTy(V: &I));
3735 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3736 setOriginForNaryOp(I);
3737 }
3738
3739 // Get an MMX-sized (64-bit) vector type, or optionally, other sized
3740 // vectors.
3741 Type *getMMXVectorTy(unsigned EltSizeInBits,
3742 unsigned X86_MMXSizeInBits = 64) {
3743 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3744 "Illegal MMX vector element size");
3745 return FixedVectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSizeInBits),
3746 NumElts: X86_MMXSizeInBits / EltSizeInBits);
3747 }
3748
3749 // Returns a signed counterpart for an (un)signed-saturate-and-pack
3750 // intrinsic.
3751 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
3752 switch (id) {
3753 case Intrinsic::x86_sse2_packsswb_128:
3754 case Intrinsic::x86_sse2_packuswb_128:
3755 return Intrinsic::x86_sse2_packsswb_128;
3756
3757 case Intrinsic::x86_sse2_packssdw_128:
3758 case Intrinsic::x86_sse41_packusdw:
3759 return Intrinsic::x86_sse2_packssdw_128;
3760
3761 case Intrinsic::x86_avx2_packsswb:
3762 case Intrinsic::x86_avx2_packuswb:
3763 return Intrinsic::x86_avx2_packsswb;
3764
3765 case Intrinsic::x86_avx2_packssdw:
3766 case Intrinsic::x86_avx2_packusdw:
3767 return Intrinsic::x86_avx2_packssdw;
3768
3769 case Intrinsic::x86_mmx_packsswb:
3770 case Intrinsic::x86_mmx_packuswb:
3771 return Intrinsic::x86_mmx_packsswb;
3772
3773 case Intrinsic::x86_mmx_packssdw:
3774 return Intrinsic::x86_mmx_packssdw;
3775
3776 case Intrinsic::x86_avx512_packssdw_512:
3777 case Intrinsic::x86_avx512_packusdw_512:
3778 return Intrinsic::x86_avx512_packssdw_512;
3779
3780 case Intrinsic::x86_avx512_packsswb_512:
3781 case Intrinsic::x86_avx512_packuswb_512:
3782 return Intrinsic::x86_avx512_packsswb_512;
3783
3784 default:
3785 llvm_unreachable("unexpected intrinsic id");
3786 }
3787 }
3788
3789 // Instrument vector pack intrinsic.
3790 //
3791 // This function instruments intrinsics like x86_mmx_packsswb, that
3792 // packs elements of 2 input vectors into half as many bits with saturation.
3793 // Shadow is propagated with the signed variant of the same intrinsic applied
3794 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3795 // MMXEltSizeInBits is used only for x86mmx arguments.
3796 //
3797 // TODO: consider using GetMinMaxUnsigned() to handle saturation precisely
3798 void handleVectorPackIntrinsic(IntrinsicInst &I,
3799 unsigned MMXEltSizeInBits = 0) {
3800 assert(I.arg_size() == 2);
3801 IRBuilder<> IRB(&I);
3802 Value *S1 = getShadow(I: &I, i: 0);
3803 Value *S2 = getShadow(I: &I, i: 1);
3804 assert(S1->getType()->isVectorTy());
3805
3806 // SExt and ICmpNE below must apply to individual elements of input vectors.
3807 // In case of x86mmx arguments, cast them to appropriate vector types and
3808 // back.
3809 Type *T =
3810 MMXEltSizeInBits ? getMMXVectorTy(EltSizeInBits: MMXEltSizeInBits) : S1->getType();
3811 if (MMXEltSizeInBits) {
3812 S1 = IRB.CreateBitCast(V: S1, DestTy: T);
3813 S2 = IRB.CreateBitCast(V: S2, DestTy: T);
3814 }
3815 Value *S1_ext =
3816 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S1, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3817 Value *S2_ext =
3818 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3819 if (MMXEltSizeInBits) {
3820 S1_ext = IRB.CreateBitCast(V: S1_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3821 S2_ext = IRB.CreateBitCast(V: S2_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3822 }
3823
3824 Value *S = IRB.CreateIntrinsic(ID: getSignedPackIntrinsic(id: I.getIntrinsicID()),
3825 Args: {S1_ext, S2_ext}, /*FMFSource=*/nullptr,
3826 Name: "_msprop_vector_pack");
3827 if (MMXEltSizeInBits)
3828 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3829 setShadow(V: &I, SV: S);
3830 setOriginForNaryOp(I);
3831 }
3832
3833 // Convert `Mask` into `<n x i1>`.
3834 Constant *createDppMask(unsigned Width, unsigned Mask) {
3835 SmallVector<Constant *, 4> R(Width);
3836 for (auto &M : R) {
3837 M = ConstantInt::getBool(Context&: F.getContext(), V: Mask & 1);
3838 Mask >>= 1;
3839 }
3840 return ConstantVector::get(V: R);
3841 }
3842
3843 // Calculate output shadow as array of booleans `<n x i1>`, assuming if any
3844 // arg is poisoned, entire dot product is poisoned.
3845 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
3846 unsigned DstMask) {
3847 const unsigned Width =
3848 cast<FixedVectorType>(Val: S->getType())->getNumElements();
3849
3850 S = IRB.CreateSelect(C: createDppMask(Width, Mask: SrcMask), True: S,
3851 False: Constant::getNullValue(Ty: S->getType()));
3852 Value *SElem = IRB.CreateOrReduce(Src: S);
3853 Value *IsClean = IRB.CreateIsNull(Arg: SElem, Name: "_msdpp");
3854 Value *DstMaskV = createDppMask(Width, Mask: DstMask);
3855
3856 return IRB.CreateSelect(
3857 C: IsClean, True: Constant::getNullValue(Ty: DstMaskV->getType()), False: DstMaskV);
3858 }
3859
3860 // See `Intel Intrinsics Guide` for `_dp_p*` instructions.
3861 //
3862 // 2 and 4 element versions produce single scalar of dot product, and then
3863 // puts it into elements of output vector, selected by 4 lowest bits of the
3864 // mask. Top 4 bits of the mask control which elements of input to use for dot
3865 // product.
3866 //
3867 // 8 element version mask still has only 4 bit for input, and 4 bit for output
3868 // mask. According to the spec it just operates as 4 element version on first
3869 // 4 elements of inputs and output, and then on last 4 elements of inputs and
3870 // output.
3871 void handleDppIntrinsic(IntrinsicInst &I) {
3872 IRBuilder<> IRB(&I);
3873
3874 Value *S0 = getShadow(I: &I, i: 0);
3875 Value *S1 = getShadow(I: &I, i: 1);
3876 Value *S = IRB.CreateOr(LHS: S0, RHS: S1);
3877
3878 const unsigned Width =
3879 cast<FixedVectorType>(Val: S->getType())->getNumElements();
3880 assert(Width == 2 || Width == 4 || Width == 8);
3881
3882 const unsigned Mask = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
3883 const unsigned SrcMask = Mask >> 4;
3884 const unsigned DstMask = Mask & 0xf;
3885
3886 // Calculate shadow as `<n x i1>`.
3887 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3888 if (Width == 8) {
3889 // First 4 elements of shadow are already calculated. `makeDppShadow`
3890 // operats on 32 bit masks, so we can just shift masks, and repeat.
3891 SI1 = IRB.CreateOr(
3892 LHS: SI1, RHS: findDppPoisonedOutput(IRB, S, SrcMask: SrcMask << 4, DstMask: DstMask << 4));
3893 }
3894 // Extend to real size of shadow, poisoning either all or none bits of an
3895 // element.
3896 S = IRB.CreateSExt(V: SI1, DestTy: S->getType(), Name: "_msdpp");
3897
3898 setShadow(V: &I, SV: S);
3899 setOriginForNaryOp(I);
3900 }
3901
3902 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
3903 C = CreateAppToShadowCast(IRB, V: C);
3904 FixedVectorType *FVT = cast<FixedVectorType>(Val: C->getType());
3905 unsigned ElSize = FVT->getElementType()->getPrimitiveSizeInBits();
3906 C = IRB.CreateAShr(LHS: C, RHS: ElSize - 1);
3907 FVT = FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: FVT->getNumElements());
3908 return IRB.CreateTrunc(V: C, DestTy: FVT);
3909 }
3910
3911 // `blendv(f, t, c)` is effectively `select(c[top_bit], t, f)`.
3912 void handleBlendvIntrinsic(IntrinsicInst &I) {
3913 Value *C = I.getOperand(i_nocapture: 2);
3914 Value *T = I.getOperand(i_nocapture: 1);
3915 Value *F = I.getOperand(i_nocapture: 0);
3916
3917 Value *Sc = getShadow(I: &I, i: 2);
3918 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
3919
3920 {
3921 IRBuilder<> IRB(&I);
3922 // Extract top bit from condition and its shadow.
3923 C = convertBlendvToSelectMask(IRB, C);
3924 Sc = convertBlendvToSelectMask(IRB, C: Sc);
3925
3926 setShadow(V: C, SV: Sc);
3927 setOrigin(V: C, Origin: Oc);
3928 }
3929
3930 handleSelectLikeInst(I, B: C, C: T, D: F);
3931 }
3932
3933 // Instrument sum-of-absolute-differences intrinsic.
3934 void handleVectorSadIntrinsic(IntrinsicInst &I, bool IsMMX = false) {
3935 const unsigned SignificantBitsPerResultElement = 16;
3936 Type *ResTy = IsMMX ? IntegerType::get(C&: *MS.C, NumBits: 64) : I.getType();
3937 unsigned ZeroBitsPerResultElement =
3938 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
3939
3940 IRBuilder<> IRB(&I);
3941 auto *Shadow0 = getShadow(I: &I, i: 0);
3942 auto *Shadow1 = getShadow(I: &I, i: 1);
3943 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3944 S = IRB.CreateBitCast(V: S, DestTy: ResTy);
3945 S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
3946 DestTy: ResTy);
3947 S = IRB.CreateLShr(LHS: S, RHS: ZeroBitsPerResultElement);
3948 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3949 setShadow(V: &I, SV: S);
3950 setOriginForNaryOp(I);
3951 }
3952
3953 // Instrument dot-product / multiply-add(-accumulate)? intrinsics.
3954 //
3955 // e.g., Two operands:
3956 // <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> %b)
3957 //
3958 // Two operands which require an EltSizeInBits override:
3959 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %a, <1 x i64> %b)
3960 //
3961 // Three operands:
3962 // <4 x i32> @llvm.x86.avx512.vpdpbusd.128
3963 // (<4 x i32> %s, <16 x i8> %a, <16 x i8> %b)
3964 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
3965 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
3966 // (these are equivalent to multiply-add on %a and %b, followed by
3967 // adding/"accumulating" %s. "Accumulation" stores the result in one
3968 // of the source registers, but this accumulate vs. add distinction
3969 // is lost when dealing with LLVM intrinsics.)
3970 //
3971 // ZeroPurifies means that multiplying a known-zero with an uninitialized
3972 // value results in an initialized value. This is applicable for integer
3973 // multiplication, but not floating-point (counter-example: NaN).
3974 void handleVectorDotProductIntrinsic(IntrinsicInst &I,
3975 unsigned ReductionFactor,
3976 bool ZeroPurifies,
3977 unsigned EltSizeInBits,
3978 enum OddOrEvenLanes Lanes) {
3979 IRBuilder<> IRB(&I);
3980
3981 [[maybe_unused]] FixedVectorType *ReturnType =
3982 cast<FixedVectorType>(Val: I.getType());
3983 assert(isa<FixedVectorType>(ReturnType));
3984
3985 // Vectors A and B, and shadows
3986 Value *Va = nullptr;
3987 Value *Vb = nullptr;
3988 Value *Sa = nullptr;
3989 Value *Sb = nullptr;
3990
3991 assert(I.arg_size() == 2 || I.arg_size() == 3);
3992 if (I.arg_size() == 2) {
3993 assert(Lanes == kBothLanes);
3994
3995 Va = I.getOperand(i_nocapture: 0);
3996 Vb = I.getOperand(i_nocapture: 1);
3997
3998 Sa = getShadow(I: &I, i: 0);
3999 Sb = getShadow(I: &I, i: 1);
4000 } else if (I.arg_size() == 3) {
4001 // Operand 0 is the accumulator. We will deal with that below.
4002 Va = I.getOperand(i_nocapture: 1);
4003 Vb = I.getOperand(i_nocapture: 2);
4004
4005 Sa = getShadow(I: &I, i: 1);
4006 Sb = getShadow(I: &I, i: 2);
4007
4008 if (Lanes == kEvenLanes || Lanes == kOddLanes) {
4009 // Convert < S0, S1, S2, S3, S4, S5, S6, S7 >
4010 // to < S0, S0, S2, S2, S4, S4, S6, S6 > (if even)
4011 // to < S1, S1, S3, S3, S5, S5, S7, S7 > (if odd)
4012 //
4013 // Note: for aarch64.neon.bfmlalb/t, the odd/even-indexed values are
4014 // zeroed, not duplicated. However, for shadow propagation, this
4015 // distinction is unimportant because Step 1 below will squeeze
4016 // each pair of elements (e.g., [S0, S0]) into a single bit, and
4017 // we only care if it is fully initialized.
4018
4019 FixedVectorType *InputShadowType = cast<FixedVectorType>(Val: Sa->getType());
4020 unsigned Width = InputShadowType->getNumElements();
4021
4022 Sa = IRB.CreateShuffleVector(
4023 V: Sa, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4024 Sb = IRB.CreateShuffleVector(
4025 V: Sb, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4026 }
4027 }
4028
4029 FixedVectorType *ParamType = cast<FixedVectorType>(Val: Va->getType());
4030 assert(ParamType == Vb->getType());
4031
4032 assert(ParamType->getPrimitiveSizeInBits() ==
4033 ReturnType->getPrimitiveSizeInBits());
4034
4035 if (I.arg_size() == 3) {
4036 [[maybe_unused]] auto *AccumulatorType =
4037 cast<FixedVectorType>(Val: I.getOperand(i_nocapture: 0)->getType());
4038 assert(AccumulatorType == ReturnType);
4039 }
4040
4041 FixedVectorType *ImplicitReturnType =
4042 cast<FixedVectorType>(Val: getShadowTy(OrigTy: ReturnType));
4043 // Step 1: instrument multiplication of corresponding vector elements
4044 if (EltSizeInBits) {
4045 ImplicitReturnType = cast<FixedVectorType>(
4046 Val: getMMXVectorTy(EltSizeInBits: EltSizeInBits * ReductionFactor,
4047 X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4048 ParamType = cast<FixedVectorType>(
4049 Val: getMMXVectorTy(EltSizeInBits, X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4050
4051 Va = IRB.CreateBitCast(V: Va, DestTy: ParamType);
4052 Vb = IRB.CreateBitCast(V: Vb, DestTy: ParamType);
4053
4054 Sa = IRB.CreateBitCast(V: Sa, DestTy: getShadowTy(OrigTy: ParamType));
4055 Sb = IRB.CreateBitCast(V: Sb, DestTy: getShadowTy(OrigTy: ParamType));
4056 } else {
4057 assert(ParamType->getNumElements() ==
4058 ReturnType->getNumElements() * ReductionFactor);
4059 }
4060
4061 // Each element of the vector is represented by a single bit (poisoned or
4062 // not) e.g., <8 x i1>.
4063 Value *SaNonZero = IRB.CreateIsNotNull(Arg: Sa);
4064 Value *SbNonZero = IRB.CreateIsNotNull(Arg: Sb);
4065 Value *And;
4066 if (ZeroPurifies) {
4067 // Multiplying an *initialized* zero by an uninitialized element results
4068 // in an initialized zero element.
4069 //
4070 // This is analogous to bitwise AND, where "AND" of 0 and a poisoned value
4071 // results in an unpoisoned value.
4072 Value *VaInt = Va;
4073 Value *VbInt = Vb;
4074 if (!Va->getType()->isIntegerTy()) {
4075 VaInt = CreateAppToShadowCast(IRB, V: Va);
4076 VbInt = CreateAppToShadowCast(IRB, V: Vb);
4077 }
4078
4079 // We check for non-zero on a per-element basis, not per-bit.
4080 Value *VaNonZero = IRB.CreateIsNotNull(Arg: VaInt);
4081 Value *VbNonZero = IRB.CreateIsNotNull(Arg: VbInt);
4082
4083 And = handleBitwiseAnd(IRB, V1: VaNonZero, V2: VbNonZero, S1: SaNonZero, S2: SbNonZero);
4084 } else {
4085 And = IRB.CreateOr(Ops: {SaNonZero, SbNonZero});
4086 }
4087
4088 // Extend <8 x i1> to <8 x i16>.
4089 // (The real pmadd intrinsic would have computed intermediate values of
4090 // <8 x i32>, but that is irrelevant for our shadow purposes because we
4091 // consider each element to be either fully initialized or fully
4092 // uninitialized.)
4093 And = IRB.CreateSExt(V: And, DestTy: Sa->getType());
4094
4095 // Step 2: instrument horizontal add
4096 // We don't need bit-precise horizontalReduce because we only want to check
4097 // if each pair/quad of elements is fully zero.
4098 // Cast to <4 x i32>.
4099 Value *Horizontal = IRB.CreateBitCast(V: And, DestTy: ImplicitReturnType);
4100
4101 // Compute <4 x i1>, then extend back to <4 x i32>.
4102 Value *OutShadow = IRB.CreateSExt(
4103 V: IRB.CreateICmpNE(LHS: Horizontal,
4104 RHS: Constant::getNullValue(Ty: Horizontal->getType())),
4105 DestTy: ImplicitReturnType);
4106
4107 // Cast it back to the required fake return type (if MMX: <1 x i64>; for
4108 // AVX, it is already correct).
4109 if (EltSizeInBits)
4110 OutShadow = CreateShadowCast(IRB, V: OutShadow, dstTy: getShadowTy(V: &I));
4111
4112 // Step 3 (if applicable): instrument accumulator
4113 if (I.arg_size() == 3)
4114 OutShadow = IRB.CreateOr(LHS: OutShadow, RHS: getShadow(I: &I, i: 0));
4115
4116 setShadow(V: &I, SV: OutShadow);
4117 setOriginForNaryOp(I);
4118 }
4119
4120 // Instrument compare-packed intrinsic.
4121 //
4122 // x86 has the predicate as the third operand, which is ImmArg e.g.,
4123 // - <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8)
4124 // - <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8)
4125 //
4126 // while Arm has separate intrinsics for >= and > e.g.,
4127 // - <2 x i32> @llvm.aarch64.neon.facge.v2i32.v2f32
4128 // (<2 x float> %A, <2 x float>)
4129 // - <2 x i32> @llvm.aarch64.neon.facgt.v2i32.v2f32
4130 // (<2 x float> %A, <2 x float>)
4131 //
4132 // Bonus: this also handles scalar cases e.g.,
4133 // - i32 @llvm.aarch64.neon.facgt.i32.f32(float %A, float %B)
4134 void handleVectorComparePackedIntrinsic(IntrinsicInst &I,
4135 bool PredicateAsOperand) {
4136 if (PredicateAsOperand) {
4137 assert(I.arg_size() == 3);
4138 assert(I.paramHasAttr(2, Attribute::ImmArg));
4139 } else
4140 assert(I.arg_size() == 2);
4141
4142 IRBuilder<> IRB(&I);
4143
4144 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
4145 // all-ones shadow.
4146 Type *ResTy = getShadowTy(V: &I);
4147 auto *Shadow0 = getShadow(I: &I, i: 0);
4148 auto *Shadow1 = getShadow(I: &I, i: 1);
4149 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4150 Value *S = IRB.CreateSExt(
4151 V: IRB.CreateICmpNE(LHS: S0, RHS: Constant::getNullValue(Ty: ResTy)), DestTy: ResTy);
4152 setShadow(V: &I, SV: S);
4153 setOriginForNaryOp(I);
4154 }
4155
4156 // Instrument compare-scalar intrinsic.
4157 // This handles both cmp* intrinsics which return the result in the first
4158 // element of a vector, and comi* which return the result as i32.
4159 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
4160 IRBuilder<> IRB(&I);
4161 auto *Shadow0 = getShadow(I: &I, i: 0);
4162 auto *Shadow1 = getShadow(I: &I, i: 1);
4163 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4164 Value *S = LowerElementShadowExtend(IRB, S: S0, T: getShadowTy(V: &I));
4165 setShadow(V: &I, SV: S);
4166 setOriginForNaryOp(I);
4167 }
4168
4169 // Instrument generic vector reduction intrinsics
4170 // by ORing together all their fields.
4171 //
4172 // If AllowShadowCast is true, the return type does not need to be the same
4173 // type as the fields
4174 // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
4175 void handleVectorReduceIntrinsic(IntrinsicInst &I, bool AllowShadowCast) {
4176 assert(I.arg_size() == 1);
4177
4178 IRBuilder<> IRB(&I);
4179 Value *S = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 0));
4180 if (AllowShadowCast)
4181 S = CreateShadowCast(IRB, V: S, dstTy: getShadowTy(V: &I));
4182 else
4183 assert(S->getType() == getShadowTy(&I));
4184 setShadow(V: &I, SV: S);
4185 setOriginForNaryOp(I);
4186 }
4187
4188 // Similar to handleVectorReduceIntrinsic but with an initial starting value.
4189 // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
4190 // %a1)
4191 // shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
4192 //
4193 // The type of the return value, initial starting value, and elements of the
4194 // vector must be identical.
4195 void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
4196 assert(I.arg_size() == 2);
4197
4198 IRBuilder<> IRB(&I);
4199 Value *Shadow0 = getShadow(I: &I, i: 0);
4200 Value *Shadow1 = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 1));
4201 assert(Shadow0->getType() == Shadow1->getType());
4202 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4203 assert(S->getType() == getShadowTy(&I));
4204 setShadow(V: &I, SV: S);
4205 setOriginForNaryOp(I);
4206 }
4207
4208 // e.g., <4 x i32> @llvm.vector.partial.reduce.add (<4 x i32>, <8 x i32>)
4209 // acc addends
4210 // <4 x fp16> @llvm.vector.partial.reduce.fadd(<4 x fp16>, <8 x fp16>)
4211 // acc addends
4212 //
4213 // output[i] = acc[i] + addend[i] + addend[i+4]
4214 //
4215 // We approximate this as "all-or-nothing" for each element of the vector
4216 // i.e., even a single uninitialized bit in acc[i], addend[i], or
4217 // addend[i+4] will make output[i] fully uninitialized.
4218 //
4219 // By converting each vector element to a single value (0 or 1) representing
4220 // whether it is uninitialized, we can apply vector.partial.reduce.add to
4221 // compute the output shadow.
4222 //
4223 // N.B. this function is ready for scalable vectors, but its dependencies
4224 // are not.
4225 void handleVectorPartialReduceAddIntrinsic(IntrinsicInst &I) {
4226 IRBuilder<> IRB(&I);
4227
4228 assert(I.arg_size() == 2);
4229 [[maybe_unused]] Value *Accumulator = I.getArgOperand(i: 0);
4230 [[maybe_unused]] Value *Addends = I.getArgOperand(i: 1);
4231 assert(isa<VectorType>(Accumulator->getType()));
4232 assert(isa<VectorType>(Addends->getType()));
4233
4234 assert(I.getType() == Accumulator->getType());
4235
4236 assert(cast<VectorType>(Addends->getType())->getElementType() ==
4237 cast<VectorType>(I.getType())->getElementType());
4238 assert(cast<VectorType>(Addends->getType())
4239 ->getElementCount()
4240 .isKnownMultipleOf(
4241 cast<VectorType>(I.getType())->getElementCount()));
4242
4243 // e.g., <4 x i32>, <8 x i32>
4244 Value *SAcc = getShadow(I: &I, i: 0);
4245 Value *SAdd = getShadow(I: &I, i: 1);
4246
4247 // All-or-nothing shadows
4248 // e.g., <4 x i1>, <8 x i1>
4249 SAcc = IRB.CreateICmpNE(LHS: SAcc, RHS: getCleanShadow(V: SAcc));
4250 SAdd = IRB.CreateICmpNE(LHS: SAdd, RHS: getCleanShadow(V: SAdd));
4251
4252 // If we applied vector.partial.reduce.add to (<M x i1>, <N x i1>) shadows,
4253 // the sums might wrap around. Zero-extend to avoid this.
4254 // 16-bit for each sum (up to 65535) ought to be enough for anybody.
4255 SAcc = IRB.CreateZExt(
4256 V: SAcc,
4257 DestTy: VectorType::get(ElementType: IRB.getInt16Ty(),
4258 EC: cast<VectorType>(Val: SAcc->getType())->getElementCount()));
4259 SAdd = IRB.CreateZExt(
4260 V: SAdd,
4261 DestTy: VectorType::get(ElementType: IRB.getInt16Ty(),
4262 EC: cast<VectorType>(Val: SAdd->getType())->getElementCount()));
4263
4264 Value *SOutput = IRB.CreateIntrinsic(
4265 RetTy: SAcc->getType(), ID: Intrinsic::vector_partial_reduce_add, Args: {SAcc, SAdd});
4266
4267 SOutput = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SOutput, RHS: getCleanShadow(V: SOutput)),
4268 DestTy: getShadowTy(V: &I));
4269
4270 setShadow(V: &I, SV: SOutput);
4271 setOriginForNaryOp(I);
4272 }
4273
4274 // Instrument vector.reduce.or intrinsic.
4275 // Valid (non-poisoned) set bits in the operand pull low the
4276 // corresponding shadow bits.
4277 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
4278 assert(I.arg_size() == 1);
4279
4280 IRBuilder<> IRB(&I);
4281 Value *OperandShadow = getShadow(I: &I, i: 0);
4282 Value *OperandUnsetBits = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
4283 Value *OperandUnsetOrPoison = IRB.CreateOr(LHS: OperandUnsetBits, RHS: OperandShadow);
4284 // Bit N is clean if any field's bit N is 1 and unpoison
4285 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandUnsetOrPoison);
4286 // Otherwise, it is clean if every field's bit N is unpoison
4287 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4288 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4289
4290 setShadow(V: &I, SV: S);
4291 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4292 }
4293
4294 // Instrument vector.reduce.and intrinsic.
4295 // Valid (non-poisoned) unset bits in the operand pull down the
4296 // corresponding shadow bits.
4297 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
4298 assert(I.arg_size() == 1);
4299
4300 IRBuilder<> IRB(&I);
4301 Value *OperandShadow = getShadow(I: &I, i: 0);
4302 Value *OperandSetOrPoison = IRB.CreateOr(LHS: I.getOperand(i_nocapture: 0), RHS: OperandShadow);
4303 // Bit N is clean if any field's bit N is 0 and unpoison
4304 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandSetOrPoison);
4305 // Otherwise, it is clean if every field's bit N is unpoison
4306 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4307 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4308
4309 setShadow(V: &I, SV: S);
4310 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4311 }
4312
4313 void handleStmxcsr(IntrinsicInst &I) {
4314 IRBuilder<> IRB(&I);
4315 Value *Addr = I.getArgOperand(i: 0);
4316 Type *Ty = IRB.getInt32Ty();
4317 Value *ShadowPtr =
4318 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment: Align(1), /*isStore*/ true).first;
4319
4320 IRB.CreateStore(Val: getCleanShadow(OrigTy: Ty), Ptr: ShadowPtr);
4321
4322 if (Opts.msan_check_access_address)
4323 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4324 }
4325
4326 void handleLdmxcsr(IntrinsicInst &I) {
4327 if (!InsertChecks)
4328 return;
4329
4330 IRBuilder<> IRB(&I);
4331 Value *Addr = I.getArgOperand(i: 0);
4332 Type *Ty = IRB.getInt32Ty();
4333 const Align Alignment = Align(1);
4334 Value *ShadowPtr, *OriginPtr;
4335 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4336 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment, /*isStore*/ false);
4337
4338 if (Opts.msan_check_access_address)
4339 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4340
4341 Value *Shadow = IRB.CreateAlignedLoad(Ty, Ptr: ShadowPtr, Align: Alignment, Name: "_ldmxcsr");
4342 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr)
4343 : getCleanOrigin();
4344 insertCheckShadow(Shadow, Origin, OrigIns: &I);
4345 }
4346
4347 void handleMaskedExpandLoad(IntrinsicInst &I) {
4348 IRBuilder<> IRB(&I);
4349 Value *Ptr = I.getArgOperand(i: 0);
4350 MaybeAlign Align = I.getParamAlign(ArgNo: 0);
4351 Value *Mask = I.getArgOperand(i: 1);
4352 Value *PassThru = I.getArgOperand(i: 2);
4353
4354 if (Opts.msan_check_access_address) {
4355 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4356 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4357 }
4358
4359 if (!PropagateShadow) {
4360 setShadow(V: &I, SV: getCleanShadow(V: &I));
4361 setOrigin(V: &I, Origin: getCleanOrigin());
4362 return;
4363 }
4364
4365 Type *ShadowTy = getShadowTy(V: &I);
4366 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4367 auto [ShadowPtr, OriginPtr] =
4368 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ false);
4369
4370 Value *Shadow =
4371 IRB.CreateMaskedExpandLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align, Mask,
4372 PassThru: getShadow(V: PassThru), Name: "_msmaskedexpload");
4373
4374 setShadow(V: &I, SV: Shadow);
4375
4376 // TODO: Store origins.
4377 setOrigin(V: &I, Origin: getCleanOrigin());
4378 }
4379
4380 void handleMaskedCompressStore(IntrinsicInst &I) {
4381 IRBuilder<> IRB(&I);
4382 Value *Values = I.getArgOperand(i: 0);
4383 Value *Ptr = I.getArgOperand(i: 1);
4384 MaybeAlign Align = I.getParamAlign(ArgNo: 1);
4385 Value *Mask = I.getArgOperand(i: 2);
4386
4387 if (Opts.msan_check_access_address) {
4388 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4389 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4390 }
4391
4392 Value *Shadow = getShadow(V: Values);
4393 Type *ElementShadowTy =
4394 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4395 auto [ShadowPtr, OriginPtrs] =
4396 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ true);
4397
4398 IRB.CreateMaskedCompressStore(Val: Shadow, Ptr: ShadowPtr, Align, Mask);
4399
4400 // TODO: Store origins.
4401 }
4402
4403 void handleMaskedGather(IntrinsicInst &I) {
4404 IRBuilder<> IRB(&I);
4405 Value *Ptrs = I.getArgOperand(i: 0);
4406 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4407 Value *Mask = I.getArgOperand(i: 1);
4408 Value *PassThru = I.getArgOperand(i: 2);
4409
4410 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4411 if (Opts.msan_check_access_address) {
4412 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4413 Value *MaskedPtrShadow = IRB.CreateSelect(
4414 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4415 Name: "_msmaskedptrs");
4416 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4417 }
4418
4419 if (!PropagateShadow) {
4420 setShadow(V: &I, SV: getCleanShadow(V: &I));
4421 setOrigin(V: &I, Origin: getCleanOrigin());
4422 return;
4423 }
4424
4425 Type *ShadowTy = getShadowTy(V: &I);
4426 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4427 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4428 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ false);
4429
4430 Value *Shadow =
4431 IRB.CreateMaskedGather(Ty: ShadowTy, Ptrs: ShadowPtrs, Alignment, Mask,
4432 PassThru: getShadow(V: PassThru), Name: "_msmaskedgather");
4433
4434 setShadow(V: &I, SV: Shadow);
4435
4436 // TODO: Store origins.
4437 setOrigin(V: &I, Origin: getCleanOrigin());
4438 }
4439
4440 void handleMaskedScatter(IntrinsicInst &I) {
4441 IRBuilder<> IRB(&I);
4442 Value *Values = I.getArgOperand(i: 0);
4443 Value *Ptrs = I.getArgOperand(i: 1);
4444 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4445 Value *Mask = I.getArgOperand(i: 2);
4446
4447 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4448 if (Opts.msan_check_access_address) {
4449 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4450 Value *MaskedPtrShadow = IRB.CreateSelect(
4451 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4452 Name: "_msmaskedptrs");
4453 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4454 }
4455
4456 Value *Shadow = getShadow(V: Values);
4457 Type *ElementShadowTy =
4458 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4459 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4460 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ true);
4461
4462 IRB.CreateMaskedScatter(Val: Shadow, Ptrs: ShadowPtrs, Alignment, Mask);
4463
4464 // TODO: Store origin.
4465 }
4466
4467 // Intrinsic::masked_store
4468 //
4469 // Note: handleAVXMaskedStore handles AVX/AVX2 variants, though AVX512 masked
4470 // stores are lowered to Intrinsic::masked_store.
4471 void handleMaskedStore(IntrinsicInst &I) {
4472 IRBuilder<> IRB(&I);
4473 Value *V = I.getArgOperand(i: 0);
4474 Value *Ptr = I.getArgOperand(i: 1);
4475 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4476 Value *Mask = I.getArgOperand(i: 2);
4477 Value *Shadow = getShadow(V);
4478
4479 if (Opts.msan_check_access_address) {
4480 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4481 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4482 }
4483
4484 Value *ShadowPtr;
4485 Value *OriginPtr;
4486 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
4487 Addr: Ptr, IRB, ShadowTy: Shadow->getType(), Alignment, /*isStore*/ true);
4488
4489 IRB.CreateMaskedStore(Val: Shadow, Ptr: ShadowPtr, Alignment, Mask);
4490
4491 if (!MS.TrackOrigins)
4492 return;
4493
4494 auto &DL = F.getDataLayout();
4495 paintOrigin(IRB, Origin: getOrigin(V), OriginPtr,
4496 TS: DL.getTypeStoreSize(Ty: Shadow->getType()),
4497 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4498 }
4499
4500 // Intrinsic::masked_load
4501 //
4502 // Note: handleAVXMaskedLoad handles AVX/AVX2 variants, though AVX512 masked
4503 // loads are lowered to Intrinsic::masked_load.
4504 void handleMaskedLoad(IntrinsicInst &I) {
4505 IRBuilder<> IRB(&I);
4506 Value *Ptr = I.getArgOperand(i: 0);
4507 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4508 Value *Mask = I.getArgOperand(i: 1);
4509 Value *PassThru = I.getArgOperand(i: 2);
4510
4511 if (Opts.msan_check_access_address) {
4512 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4513 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4514 }
4515
4516 if (!PropagateShadow) {
4517 setShadow(V: &I, SV: getCleanShadow(V: &I));
4518 setOrigin(V: &I, Origin: getCleanOrigin());
4519 return;
4520 }
4521
4522 Type *ShadowTy = getShadowTy(V: &I);
4523 Value *ShadowPtr, *OriginPtr;
4524 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4525 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
4526 setShadow(V: &I, SV: IRB.CreateMaskedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Alignment, Mask,
4527 PassThru: getShadow(V: PassThru), Name: "_msmaskedld"));
4528
4529 if (!MS.TrackOrigins)
4530 return;
4531
4532 // Choose between PassThru's and the loaded value's origins.
4533 Value *MaskedPassThruShadow = IRB.CreateAnd(
4534 LHS: getShadow(V: PassThru), RHS: IRB.CreateSExt(V: IRB.CreateNeg(V: Mask), DestTy: ShadowTy));
4535
4536 Value *NotNull = convertToBool(V: MaskedPassThruShadow, IRB, name: "_mscmp");
4537
4538 Value *PtrOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr);
4539 Value *Origin = IRB.CreateSelect(C: NotNull, True: getOrigin(V: PassThru), False: PtrOrigin);
4540
4541 setOrigin(V: &I, Origin);
4542 }
4543
4544 // e.g., <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> %dividend,
4545 // <4 x i32> %divisor,
4546 // <4 x i1> %mask)
4547 //
4548 // As handleIntegerDiv(), but per-lane: strict on the divisor and propagating
4549 // the dividend, both only on the enabled lanes. Disabled lanes cannot cause
4550 // undefined behaviour, and their result is poison.
4551 void handleMaskedIntegerDivRem(IntrinsicInst &I) {
4552 assert(I.arg_size() == 3);
4553 IRBuilder<> IRB(&I);
4554 Value *Dividend = I.getArgOperand(i: 0);
4555 Value *Divisor = I.getArgOperand(i: 1);
4556 Value *Mask = I.getArgOperand(i: 2);
4557
4558 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4559
4560 Value *MaskedDivisorShadow = IRB.CreateSelect(
4561 C: Mask, True: getShadow(V: Divisor), False: getCleanShadow(V: Divisor), Name: "_msmaskeddivisor");
4562 insertCheckShadow(Shadow: MaskedDivisorShadow, Origin: getOrigin(V: Divisor), OrigIns: &I);
4563
4564 if (!PropagateShadow) {
4565 setShadow(V: &I, SV: getCleanShadow(V: &I));
4566 setOrigin(V: &I, Origin: getCleanOrigin());
4567 return;
4568 }
4569
4570 setShadow(V: &I, SV: IRB.CreateSelect(C: Mask, True: getShadow(V: Dividend),
4571 False: getPoisonedShadow(V: &I), Name: "_msmaskeddiv"));
4572 setOrigin(V: &I, Origin: getOrigin(V: Dividend));
4573 }
4574
4575 // e.g., void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)
4576 // dst mask src
4577 //
4578 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4579 // by handleMaskedStore.
4580 //
4581 // This function handles AVX and AVX2 masked stores; these use the MSBs of a
4582 // vector of integers, unlike the LLVM masked intrinsics, which require a
4583 // vector of booleans. X86InstCombineIntrinsic.cpp::simplifyX86MaskedLoad
4584 // mentions that the x86 backend does not know how to efficiently convert
4585 // from a vector of booleans back into the AVX mask format; therefore, they
4586 // (and we) do not reduce AVX/AVX2 masked intrinsics into LLVM masked
4587 // intrinsics.
4588 void handleAVXMaskedStore(IntrinsicInst &I) {
4589 assert(I.arg_size() == 3);
4590
4591 IRBuilder<> IRB(&I);
4592
4593 Value *Dst = I.getArgOperand(i: 0);
4594 assert(Dst->getType()->isPointerTy() && "Destination is not a pointer!");
4595
4596 Value *Mask = I.getArgOperand(i: 1);
4597 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4598
4599 Value *Src = I.getArgOperand(i: 2);
4600 assert(isa<VectorType>(Src->getType()) && "Source is not a vector!");
4601
4602 const Align Alignment = Align(1);
4603
4604 Value *SrcShadow = getShadow(V: Src);
4605
4606 if (Opts.msan_check_access_address) {
4607 insertCheckShadowOf(Val: Dst, OrigIns: &I);
4608 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4609 }
4610
4611 Value *DstShadowPtr;
4612 Value *DstOriginPtr;
4613 std::tie(args&: DstShadowPtr, args&: DstOriginPtr) = getShadowOriginPtr(
4614 Addr: Dst, IRB, ShadowTy: SrcShadow->getType(), Alignment, /*isStore*/ true);
4615
4616 SmallVector<Value *, 2> ShadowArgs;
4617 ShadowArgs.append(NumInputs: 1, Elt: DstShadowPtr);
4618 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4619 // The intrinsic may require floating-point but shadows can be arbitrary
4620 // bit patterns, of which some would be interpreted as "invalid"
4621 // floating-point values (NaN etc.); we assume the intrinsic will happily
4622 // copy them.
4623 ShadowArgs.append(NumInputs: 1, Elt: IRB.CreateBitCast(V: SrcShadow, DestTy: Src->getType()));
4624
4625 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4626 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4627 setShadow(V: &I, SV: CI);
4628
4629 if (!MS.TrackOrigins)
4630 return;
4631
4632 // Approximation only
4633 auto &DL = F.getDataLayout();
4634 paintOrigin(IRB, Origin: getOrigin(V: Src), OriginPtr: DstOriginPtr,
4635 TS: DL.getTypeStoreSize(Ty: SrcShadow->getType()),
4636 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4637 }
4638
4639 // e.g., <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)
4640 // return src mask
4641 //
4642 // Masked-off values are replaced with 0, which conveniently also represents
4643 // initialized memory.
4644 //
4645 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4646 // by handleMaskedStore.
4647 //
4648 // We do not combine this with handleMaskedLoad; see comment in
4649 // handleAVXMaskedStore for the rationale.
4650 //
4651 // This is subtly different than handleIntrinsicByApplyingToShadow(I, 1)
4652 // because we need to apply getShadowOriginPtr, not getShadow, to the first
4653 // parameter.
4654 void handleAVXMaskedLoad(IntrinsicInst &I) {
4655 assert(I.arg_size() == 2);
4656
4657 IRBuilder<> IRB(&I);
4658
4659 Value *Src = I.getArgOperand(i: 0);
4660 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
4661
4662 Value *Mask = I.getArgOperand(i: 1);
4663 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4664
4665 const Align Alignment = Align(1);
4666
4667 if (Opts.msan_check_access_address) {
4668 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4669 }
4670
4671 Type *SrcShadowTy = getShadowTy(V: Src);
4672 Value *SrcShadowPtr, *SrcOriginPtr;
4673 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
4674 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment, /*isStore*/ false);
4675
4676 SmallVector<Value *, 2> ShadowArgs;
4677 ShadowArgs.append(NumInputs: 1, Elt: SrcShadowPtr);
4678 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4679
4680 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4681 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4682 // The AVX masked load intrinsics do not have integer variants. We use the
4683 // floating-point variants, which will happily copy the shadows even if
4684 // they are interpreted as "invalid" floating-point values (NaN etc.).
4685 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4686
4687 if (!MS.TrackOrigins)
4688 return;
4689
4690 // The "pass-through" value is always zero (initialized). To the extent
4691 // that that results in initialized aligned 4-byte chunks, the origin value
4692 // is ignored. It is therefore correct to simply copy the origin from src.
4693 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
4694 setOrigin(V: &I, Origin: PtrSrcOrigin);
4695 }
4696
4697 // Test whether the mask indices are initialized, only checking the bits that
4698 // are actually used.
4699 //
4700 // e.g., if Idx is <32 x i16>, only (log2(32) == 5) bits of each index are
4701 // used/checked.
4702 void maskedCheckAVXIndexShadow(IRBuilder<> &IRB, Value *Idx, Instruction *I) {
4703 assert(isFixedIntVector(Idx));
4704 auto IdxVectorSize =
4705 cast<FixedVectorType>(Val: Idx->getType())->getNumElements();
4706 assert(isPowerOf2_64(IdxVectorSize));
4707
4708 // Compiler isn't smart enough, let's help it
4709 if (isa<Constant>(Val: Idx))
4710 return;
4711
4712 auto *IdxShadow = getShadow(V: Idx);
4713 Value *Truncated = IRB.CreateTrunc(
4714 V: IdxShadow,
4715 DestTy: FixedVectorType::get(ElementType: Type::getIntNTy(C&: *MS.C, N: Log2_64(Value: IdxVectorSize)),
4716 NumElts: IdxVectorSize));
4717 insertCheckShadow(Shadow: Truncated, Origin: getOrigin(V: Idx), OrigIns: I);
4718 }
4719
4720 // Instrument AVX permutation intrinsic.
4721 // We apply the same permutation (argument index 1) to the shadow.
4722 void handleAVXVpermilvar(IntrinsicInst &I) {
4723 IRBuilder<> IRB(&I);
4724 Value *Shadow = getShadow(I: &I, i: 0);
4725 maskedCheckAVXIndexShadow(IRB, Idx: I.getArgOperand(i: 1), I: &I);
4726
4727 // Shadows are integer-ish types but some intrinsics require a
4728 // different (e.g., floating-point) type.
4729 Shadow = IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i: 0)->getType());
4730 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4731 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Shadow, I.getArgOperand(i: 1)});
4732
4733 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4734 setOriginForNaryOp(I);
4735 }
4736
4737 // Instrument AVX permutation intrinsic.
4738 // We apply the same permutation (argument index 1) to the shadows.
4739 void handleAVXVpermi2var(IntrinsicInst &I) {
4740 assert(I.arg_size() == 3);
4741 assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
4742 assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
4743 assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
4744 [[maybe_unused]] auto ArgVectorSize =
4745 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4746 assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
4747 ->getNumElements() == ArgVectorSize);
4748 assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
4749 ->getNumElements() == ArgVectorSize);
4750 assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
4751 assert(I.getType() == I.getArgOperand(0)->getType());
4752 assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
4753 IRBuilder<> IRB(&I);
4754 Value *AShadow = getShadow(I: &I, i: 0);
4755 Value *Idx = I.getArgOperand(i: 1);
4756 Value *BShadow = getShadow(I: &I, i: 2);
4757
4758 maskedCheckAVXIndexShadow(IRB, Idx, I: &I);
4759
4760 // Shadows are integer-ish types but some intrinsics require a
4761 // different (e.g., floating-point) type.
4762 AShadow = IRB.CreateBitCast(V: AShadow, DestTy: I.getArgOperand(i: 0)->getType());
4763 BShadow = IRB.CreateBitCast(V: BShadow, DestTy: I.getArgOperand(i: 2)->getType());
4764 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4765 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {AShadow, Idx, BShadow});
4766 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4767 setOriginForNaryOp(I);
4768 }
4769
4770 [[maybe_unused]] static bool isFixedIntVectorTy(const Type *T) {
4771 return isa<FixedVectorType>(Val: T) && T->isIntOrIntVectorTy();
4772 }
4773
4774 [[maybe_unused]] static bool isFixedFPVectorTy(const Type *T) {
4775 return isa<FixedVectorType>(Val: T) && T->isFPOrFPVectorTy();
4776 }
4777
4778 [[maybe_unused]] static bool isFixedIntVector(const Value *V) {
4779 return isFixedIntVectorTy(T: V->getType());
4780 }
4781
4782 [[maybe_unused]] static bool isFixedFPVector(const Value *V) {
4783 return isFixedFPVectorTy(T: V->getType());
4784 }
4785
4786 // e.g., <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
4787 // (<16 x float> a, <16 x i32> writethru, i16 mask,
4788 // i32 rounding)
4789 //
4790 // Inconveniently, some similar intrinsics have a different operand order:
4791 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
4792 // (<16 x float> a, i32 rounding, <16 x i16> writethru,
4793 // i16 mask)
4794 //
4795 // If the return type has more elements than A, the excess elements are
4796 // zeroed (and the corresponding shadow is initialized).
4797 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
4798 // (<4 x float> a, i32 rounding, <8 x i16> writethru,
4799 // i8 mask)
4800 //
4801 // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
4802 // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
4803 // where all_or_nothing(x) is fully uninitialized if x has any
4804 // uninitialized bits
4805 void handleAVX512VectorConvertFPToInt(IntrinsicInst &I, bool LastMask) {
4806 IRBuilder<> IRB(&I);
4807
4808 assert(I.arg_size() == 4);
4809 Value *A = I.getOperand(i_nocapture: 0);
4810 Value *WriteThrough;
4811 Value *Mask;
4812 Value *RoundingMode;
4813 if (LastMask) {
4814 WriteThrough = I.getOperand(i_nocapture: 2);
4815 Mask = I.getOperand(i_nocapture: 3);
4816 RoundingMode = I.getOperand(i_nocapture: 1);
4817 } else {
4818 WriteThrough = I.getOperand(i_nocapture: 1);
4819 Mask = I.getOperand(i_nocapture: 2);
4820 RoundingMode = I.getOperand(i_nocapture: 3);
4821 }
4822
4823 assert(isFixedFPVector(A));
4824 assert(isFixedIntVector(WriteThrough));
4825
4826 unsigned ANumElements =
4827 cast<FixedVectorType>(Val: A->getType())->getNumElements();
4828 [[maybe_unused]] unsigned WriteThruNumElements =
4829 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
4830 assert(ANumElements == WriteThruNumElements ||
4831 ANumElements * 2 == WriteThruNumElements);
4832
4833 assert(Mask->getType()->isIntegerTy());
4834 unsigned MaskNumElements = Mask->getType()->getScalarSizeInBits();
4835 assert(ANumElements == MaskNumElements ||
4836 ANumElements * 2 == MaskNumElements);
4837
4838 assert(WriteThruNumElements == MaskNumElements);
4839
4840 // Some bits of the mask may be unused, though it's unusual to have partly
4841 // uninitialized bits.
4842 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4843
4844 assert(RoundingMode->getType()->isIntegerTy());
4845 // Only some bits of the rounding mode are used, though it's very
4846 // unusual to have uninitialized bits there (more commonly, it's a
4847 // constant).
4848 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
4849
4850 assert(I.getType() == WriteThrough->getType());
4851
4852 Value *AShadow = getShadow(V: A);
4853 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
4854
4855 if (ANumElements * 2 == MaskNumElements) {
4856 // Ensure that the irrelevant bits of the mask are zero, hence selecting
4857 // from the zeroed shadow instead of the writethrough's shadow.
4858 Mask =
4859 IRB.CreateTrunc(V: Mask, DestTy: IRB.getIntNTy(N: ANumElements), Name: "_ms_mask_trunc");
4860 Mask =
4861 IRB.CreateZExt(V: Mask, DestTy: IRB.getIntNTy(N: MaskNumElements), Name: "_ms_mask_zext");
4862 }
4863
4864 // Convert i16 mask to <16 x i1>
4865 Mask = IRB.CreateBitCast(
4866 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: MaskNumElements),
4867 Name: "_ms_mask_bitcast");
4868
4869 /// For floating-point to integer conversion, the output is:
4870 /// - fully uninitialized if *any* bit of the input is uninitialized
4871 /// - fully ininitialized if all bits of the input are ininitialized
4872 /// We apply the same principle on a per-element basis for vectors.
4873 ///
4874 /// We use the scalar width of the return type instead of A's.
4875 AShadow = IRB.CreateSExt(
4876 V: IRB.CreateICmpNE(LHS: AShadow, RHS: getCleanShadow(OrigTy: AShadow->getType())),
4877 DestTy: getShadowTy(V: &I), Name: "_ms_a_shadow");
4878
4879 Value *WriteThroughShadow = getShadow(V: WriteThrough);
4880 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow,
4881 Name: "_ms_writethru_select");
4882
4883 setShadow(V: &I, SV: Shadow);
4884 setOriginForNaryOp(I);
4885 }
4886
4887 static SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
4888 SmallVector<int, 8> Mask;
4889 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
4890 Mask.append(NumInputs: 2, Elt: X);
4891 }
4892 return Mask;
4893 }
4894
4895 // Instrument pclmul intrinsics.
4896 // These intrinsics operate either on odd or on even elements of the input
4897 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
4898 // Replace the unused elements with copies of the used ones, ex:
4899 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
4900 // or
4901 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
4902 // and then apply the usual shadow combining logic.
4903 void handlePclmulIntrinsic(IntrinsicInst &I) {
4904 IRBuilder<> IRB(&I);
4905 unsigned Width =
4906 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4907 assert(isa<ConstantInt>(I.getArgOperand(2)) &&
4908 "pclmul 3rd operand must be a constant");
4909 unsigned Imm = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4910 Value *Shuf0 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 0),
4911 Mask: getPclmulMask(Width, OddElements: Imm & 0x01));
4912 Value *Shuf1 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 1),
4913 Mask: getPclmulMask(Width, OddElements: Imm & 0x10));
4914 ShadowAndOriginCombiner SOC(this, IRB);
4915 SOC.Add(OpShadow: Shuf0, OpOrigin: getOrigin(I: &I, i: 0));
4916 SOC.Add(OpShadow: Shuf1, OpOrigin: getOrigin(I: &I, i: 1));
4917 SOC.Done(I: &I);
4918 }
4919
4920 // Instrument _mm_*_sd|ss intrinsics
4921 void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
4922 IRBuilder<> IRB(&I);
4923 unsigned Width =
4924 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4925 Value *First = getShadow(I: &I, i: 0);
4926 Value *Second = getShadow(I: &I, i: 1);
4927 // First element of second operand, remaining elements of first operand
4928 SmallVector<int, 16> Mask;
4929 Mask.push_back(Elt: Width);
4930 for (unsigned i = 1; i < Width; i++)
4931 Mask.push_back(Elt: i);
4932 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: Second, Mask);
4933
4934 setShadow(V: &I, SV: Shadow);
4935 setOriginForNaryOp(I);
4936 }
4937
4938 void handleVtestIntrinsic(IntrinsicInst &I) {
4939 IRBuilder<> IRB(&I);
4940 Value *Shadow0 = getShadow(I: &I, i: 0);
4941 Value *Shadow1 = getShadow(I: &I, i: 1);
4942 Value *Or = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4943 Value *NZ = IRB.CreateICmpNE(LHS: Or, RHS: Constant::getNullValue(Ty: Or->getType()));
4944 Value *Scalar = convertShadowToScalar(V: NZ, IRB);
4945 Value *Shadow = IRB.CreateZExt(V: Scalar, DestTy: getShadowTy(V: &I));
4946
4947 setShadow(V: &I, SV: Shadow);
4948 setOriginForNaryOp(I);
4949 }
4950
4951 void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
4952 IRBuilder<> IRB(&I);
4953 unsigned Width =
4954 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4955 Value *First = getShadow(I: &I, i: 0);
4956 Value *Second = getShadow(I: &I, i: 1);
4957 Value *OrShadow = IRB.CreateOr(LHS: First, RHS: Second);
4958 // First element of both OR'd together, remaining elements of first operand
4959 SmallVector<int, 16> Mask;
4960 Mask.push_back(Elt: Width);
4961 for (unsigned i = 1; i < Width; i++)
4962 Mask.push_back(Elt: i);
4963 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: OrShadow, Mask);
4964
4965 setShadow(V: &I, SV: Shadow);
4966 setOriginForNaryOp(I);
4967 }
4968
4969 // _mm_round_ps / _mm_round_ps.
4970 // Similar to maybeHandleSimpleNomemIntrinsic except
4971 // the second argument is guaranteed to be a constant integer.
4972 void handleRoundPdPsIntrinsic(IntrinsicInst &I) {
4973 assert(I.getArgOperand(0)->getType() == I.getType());
4974 assert(I.arg_size() == 2);
4975 assert(isa<ConstantInt>(I.getArgOperand(1)));
4976
4977 IRBuilder<> IRB(&I);
4978 ShadowAndOriginCombiner SC(this, IRB);
4979 SC.Add(V: I.getArgOperand(i: 0));
4980 SC.Done(I: &I);
4981 }
4982
4983 // Instrument @llvm.abs intrinsic.
4984 //
4985 // e.g., i32 @llvm.abs.i32 (i32 <Src>, i1 <is_int_min_poison>)
4986 // <4 x i32> @llvm.abs.v4i32(<4 x i32> <Src>, i1 <is_int_min_poison>)
4987 void handleAbsIntrinsic(IntrinsicInst &I) {
4988 assert(I.arg_size() == 2);
4989 Value *Src = I.getArgOperand(i: 0);
4990 Value *IsIntMinPoison = I.getArgOperand(i: 1);
4991
4992 assert(I.getType()->isIntOrIntVectorTy());
4993
4994 assert(Src->getType() == I.getType());
4995
4996 assert(IsIntMinPoison->getType()->isIntegerTy());
4997 assert(IsIntMinPoison->getType()->getIntegerBitWidth() == 1);
4998
4999 IRBuilder<> IRB(&I);
5000 Value *SrcShadow = getShadow(V: Src);
5001
5002 APInt MinVal =
5003 APInt::getSignedMinValue(numBits: Src->getType()->getScalarSizeInBits());
5004 Value *MinValVec = ConstantInt::get(Ty: Src->getType(), V: MinVal);
5005 Value *SrcIsMin = IRB.CreateICmp(P: CmpInst::ICMP_EQ, LHS: Src, RHS: MinValVec);
5006
5007 Value *PoisonedShadow = getPoisonedShadow(V: Src);
5008 Value *PoisonedIfIntMinShadow =
5009 IRB.CreateSelect(C: SrcIsMin, True: PoisonedShadow, False: SrcShadow);
5010 Value *Shadow =
5011 IRB.CreateSelect(C: IsIntMinPoison, True: PoisonedIfIntMinShadow, False: SrcShadow);
5012
5013 setShadow(V: &I, SV: Shadow);
5014 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5015 }
5016
5017 void handleIsFpClass(IntrinsicInst &I) {
5018 IRBuilder<> IRB(&I);
5019 Value *Shadow = getShadow(I: &I, i: 0);
5020 setShadow(V: &I, SV: IRB.CreateICmpNE(LHS: Shadow, RHS: getCleanShadow(V: Shadow)));
5021 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5022 }
5023
5024 void handleArithmeticWithOverflow(IntrinsicInst &I) {
5025 IRBuilder<> IRB(&I);
5026 Value *Shadow0 = getShadow(I: &I, i: 0);
5027 Value *Shadow1 = getShadow(I: &I, i: 1);
5028 Value *ShadowElt0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
5029 Value *ShadowElt1 =
5030 IRB.CreateICmpNE(LHS: ShadowElt0, RHS: getCleanShadow(V: ShadowElt0));
5031
5032 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5033 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt0, Idxs: 0);
5034 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt1, Idxs: 1);
5035
5036 setShadow(V: &I, SV: Shadow);
5037 setOriginForNaryOp(I);
5038 }
5039
5040 void handleModfOrSincos(IntrinsicInst &I) {
5041 IRBuilder<> IRB(&I);
5042 Value *ArgShadow = getShadow(I: &I, i: 0);
5043 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5044 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 0);
5045 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 1);
5046 setShadow(V: &I, SV: Shadow);
5047 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5048 }
5049
5050 Value *extractLowerShadow(IRBuilder<> &IRB, Value *V) {
5051 assert(isa<FixedVectorType>(V->getType()));
5052 assert(cast<FixedVectorType>(V->getType())->getNumElements() > 0);
5053 Value *Shadow = getShadow(V);
5054 return IRB.CreateExtractElement(Vec: Shadow,
5055 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5056 }
5057
5058 // Handle llvm.x86.avx512.mask.pmov{,s,us}.*.{128,256,512}
5059 //
5060 // e.g., call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512
5061 // (<8 x i64>, <16 x i8>, i8)
5062 // A WriteThru Mask
5063 //
5064 // call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512
5065 // (<16 x i32>, <16 x i8>, i16)
5066 //
5067 // Dst[i] = Mask[i] ? truncate_or_saturate(A[i]) : WriteThru[i]
5068 // Dst_shadow[i] = Mask[i] ? truncate(A_shadow[i]) : WriteThru_shadow[i]
5069 //
5070 // If Dst has more elements than A, the excess elements are zeroed (and the
5071 // corresponding shadow is initialized).
5072 //
5073 // Note: for PMOV (truncation), handleIntrinsicByApplyingToShadow is precise
5074 // and is much faster than this handler.
5075 void handleAVX512VectorDownConvert(IntrinsicInst &I) {
5076 IRBuilder<> IRB(&I);
5077
5078 assert(I.arg_size() == 3);
5079 Value *A = I.getOperand(i_nocapture: 0);
5080 Value *WriteThrough = I.getOperand(i_nocapture: 1);
5081 Value *Mask = I.getOperand(i_nocapture: 2);
5082
5083 assert(isFixedIntVector(A));
5084 assert(isFixedIntVector(WriteThrough));
5085
5086 unsigned ANumElements =
5087 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5088 unsigned OutputNumElements =
5089 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
5090 assert(ANumElements == OutputNumElements ||
5091 ANumElements * 2 == OutputNumElements);
5092 // N.B. some PMOV{,S,US} instructions have a 4x or even 8x ratio in the
5093 // number of elements e.g.,
5094 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256
5095 // (<4 x i64>, <16 x i8>, i8)
5096 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128
5097 // (<2 x i64>, <16 x i8>, i8)
5098 // However, we currently handle those elsewhere.
5099
5100 assert(Mask->getType()->isIntegerTy());
5101 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5102
5103 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5104 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5105 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5106 assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
5107
5108 assert(I.getType() == WriteThrough->getType());
5109
5110 // Widen the mask, if necessary, to have one bit per element of the output
5111 // vector.
5112 // We want the extra bits to have '1's, so that the CreateSelect will
5113 // select the values from AShadow instead of WriteThroughShadow ("maskless"
5114 // versions of the intrinsics are sometimes implemented using an all-1's
5115 // mask and an undefined value for WriteThroughShadow). We accomplish this
5116 // by using bitwise NOT before and after the ZExt.
5117 if (ANumElements != OutputNumElements) {
5118 Mask = IRB.CreateNot(V: Mask);
5119 Mask = IRB.CreateZExt(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements),
5120 Name: "_ms_widen_mask");
5121 Mask = IRB.CreateNot(V: Mask);
5122 }
5123 Mask = IRB.CreateBitCast(
5124 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5125
5126 Value *AShadow = getShadow(V: A);
5127
5128 // The return type might have more elements than the input.
5129 // Temporarily shrink the return type's number of elements.
5130 VectorType *ShadowType = maybeShrinkVectorShadowType(Src: A, I);
5131
5132 // PMOV truncates; PMOVS/PMOVUS uses signed/unsigned saturation.
5133 // This handler treats them all as truncation, which leads to some rare
5134 // false positives in the cases where the truncated bytes could
5135 // unambiguously saturate the value e.g., if A = ??????10 ????????
5136 // (big-endian), the unsigned saturated byte conversion is 11111111 i.e.,
5137 // fully defined, but the truncated byte is ????????.
5138 //
5139 // TODO: use GetMinMaxUnsigned() to handle saturation precisely.
5140 AShadow = IRB.CreateTrunc(V: AShadow, DestTy: ShadowType, Name: "_ms_trunc_shadow");
5141 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
5142
5143 Value *WriteThroughShadow = getShadow(V: WriteThrough);
5144
5145 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow);
5146 setShadow(V: &I, SV: Shadow);
5147 setOriginForNaryOp(I);
5148 }
5149
5150 // Handle llvm.x86.avx512.* instructions that take vector(s) of floating-point
5151 // values and perform an operation whose shadow propagation should be handled
5152 // as all-or-nothing [*], with masking provided by a vector and a mask
5153 // supplied as an integer.
5154 //
5155 // [*] if all bits of a vector element are initialized, the output is fully
5156 // initialized; otherwise, the output is fully uninitialized
5157 //
5158 // e.g., <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
5159 // (<16 x float>, <16 x float>, i16)
5160 // A WriteThru Mask
5161 //
5162 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
5163 // (<2 x double>, <2 x double>, i8)
5164 // A WriteThru Mask
5165 //
5166 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
5167 // (<8 x double>, i32, <8 x double>, i8, i32)
5168 // A Imm WriteThru Mask Rounding
5169 //
5170 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
5171 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
5172 // WriteThru A B Mask Rnd
5173 //
5174 // All operands other than A, B, ..., and WriteThru (e.g., Mask, Imm,
5175 // Rounding) must be fully initialized.
5176 //
5177 // Dst[i] = Mask[i] ? some_op(A[i], B[i], ...)
5178 // : WriteThru[i]
5179 // Dst_shadow[i] = Mask[i] ? all_or_nothing(A_shadow[i] | B_shadow[i] | ...)
5180 // : WriteThru_shadow[i]
5181 void handleAVX512VectorGenericMaskedFP(IntrinsicInst &I,
5182 SmallVector<unsigned, 4> DataIndices,
5183 unsigned WriteThruIndex,
5184 unsigned MaskIndex) {
5185 IRBuilder<> IRB(&I);
5186
5187 unsigned NumArgs = I.arg_size();
5188
5189 assert(WriteThruIndex < NumArgs);
5190 assert(MaskIndex < NumArgs);
5191 assert(WriteThruIndex != MaskIndex);
5192 Value *WriteThru = I.getOperand(i_nocapture: WriteThruIndex);
5193
5194 unsigned OutputNumElements =
5195 cast<FixedVectorType>(Val: WriteThru->getType())->getNumElements();
5196
5197 assert(DataIndices.size() > 0);
5198
5199 bool isData[16] = {false};
5200 assert(NumArgs <= 16);
5201 for (unsigned i : DataIndices) {
5202 assert(i < NumArgs);
5203 assert(i != WriteThruIndex);
5204 assert(i != MaskIndex);
5205
5206 isData[i] = true;
5207
5208 Value *A = I.getOperand(i_nocapture: i);
5209 assert(isFixedFPVector(A));
5210 [[maybe_unused]] unsigned ANumElements =
5211 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5212 assert(ANumElements == OutputNumElements);
5213 }
5214
5215 Value *Mask = I.getOperand(i_nocapture: MaskIndex);
5216
5217 assert(isFixedFPVector(WriteThru));
5218
5219 for (unsigned i = 0; i < NumArgs; ++i) {
5220 if (!isData[i] && i != WriteThruIndex) {
5221 // Imm, Mask, Rounding etc. are "control" data, hence we require that
5222 // they be fully initialized.
5223 assert(I.getOperand(i)->getType()->isIntegerTy());
5224 insertCheckShadowOf(Val: I.getOperand(i_nocapture: i), OrigIns: &I);
5225 }
5226 }
5227
5228 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5229 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5230 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5231 assert(Mask->getType()->getScalarSizeInBits() == OutputNumElements);
5232
5233 assert(I.getType() == WriteThru->getType());
5234
5235 Mask = IRB.CreateBitCast(
5236 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5237
5238 Value *DataShadow = nullptr;
5239 for (unsigned i : DataIndices) {
5240 Value *A = I.getOperand(i_nocapture: i);
5241 if (DataShadow)
5242 DataShadow = IRB.CreateOr(LHS: DataShadow, RHS: getShadow(V: A));
5243 else
5244 DataShadow = getShadow(V: A);
5245 }
5246
5247 // All-or-nothing shadow
5248 DataShadow =
5249 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: DataShadow, RHS: getCleanShadow(V: DataShadow)),
5250 DestTy: DataShadow->getType());
5251
5252 Value *WriteThruShadow = getShadow(V: WriteThru);
5253
5254 Value *Shadow = IRB.CreateSelect(C: Mask, True: DataShadow, False: WriteThruShadow);
5255 setShadow(V: &I, SV: Shadow);
5256
5257 setOriginForNaryOp(I);
5258 }
5259
5260 // AVX512 Floating-Point Classification
5261 //
5262 // e.g.,
5263 // - < 8 x i1> @llvm.x86.avx512.fpclass.pd.512
5264 // (<8 x double> %input, i32 %classifiers)
5265 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512
5266 // (<16 x float> %input, i32 %classifiers)
5267 void handleAVX512FPClass(IntrinsicInst &I) {
5268 IRBuilder<> IRB(&I);
5269
5270 assert(I.arg_size() == 2);
5271
5272 Value *Input = I.getOperand(i_nocapture: 0);
5273 assert(isFixedFPVector(Input));
5274 [[maybe_unused]] FixedVectorType *InputType = cast<FixedVectorType>(Val: Input->getType());
5275
5276 Value *Classifiers = I.getOperand(i_nocapture: 1);
5277 assert(isa<ConstantInt>(Classifiers));
5278 // No shadow check needed for constants
5279
5280 assert(isFixedIntVectorTy(I.getType()));
5281 FixedVectorType *OutputType = cast<FixedVectorType>(Val: I.getType());
5282 assert(OutputType->getScalarSizeInBits() == 1);
5283
5284 assert(OutputType->getNumElements() == InputType->getNumElements());
5285
5286 Value *OutputShadow;
5287 if (cast<ConstantInt>(Val: Classifiers)->isZero())
5288 // Each bit specifies whether a particular classifier is enabled.
5289 // If Classifiers == 0, the output is trivially known to be zero, thus
5290 // the output is fully initialized.
5291 OutputShadow = getCleanShadow(OrigTy: OutputType);
5292 else
5293 // Approximate each bit of the output shadow based on whether the
5294 // corresponding input element is fully initialized. It is only
5295 // approximate because some classifications do not rely on all bits of
5296 // the input element.
5297 OutputShadow = IRB.CreateICmpNE(LHS: getShadow(V: Input), RHS: getCleanShadow(V: Input));
5298
5299 setShadow(V: &I, SV: OutputShadow);
5300
5301 setOriginForNaryOp(I);
5302 }
5303
5304 // For sh.* compiler intrinsics:
5305 // llvm.x86.avx512fp16.mask.{add/sub/mul/div/max/min}.sh.round
5306 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
5307 // A B WriteThru Mask RoundingMode
5308 //
5309 // DstShadow[0] = Mask[0] ? (AShadow[0] | BShadow[0]) : WriteThruShadow[0]
5310 // DstShadow[1..7] = AShadow[1..7]
5311 void visitGenericScalarHalfwordInst(IntrinsicInst &I) {
5312 IRBuilder<> IRB(&I);
5313
5314 assert(I.arg_size() == 5);
5315 Value *A = I.getOperand(i_nocapture: 0);
5316 Value *B = I.getOperand(i_nocapture: 1);
5317 Value *WriteThrough = I.getOperand(i_nocapture: 2);
5318 Value *Mask = I.getOperand(i_nocapture: 3);
5319 Value *RoundingMode = I.getOperand(i_nocapture: 4);
5320
5321 // Technically, we could probably just check whether the LSB is
5322 // initialized, but intuitively it feels like a partly uninitialized mask
5323 // is unintended, and we should warn the user immediately.
5324 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5325 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
5326
5327 assert(isa<FixedVectorType>(A->getType()));
5328 unsigned NumElements =
5329 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5330 assert(NumElements == 8);
5331 assert(A->getType() == B->getType());
5332 assert(B->getType() == WriteThrough->getType());
5333 assert(Mask->getType()->getPrimitiveSizeInBits() == NumElements);
5334 assert(RoundingMode->getType()->isIntegerTy());
5335
5336 Value *ALowerShadow = extractLowerShadow(IRB, V: A);
5337 Value *BLowerShadow = extractLowerShadow(IRB, V: B);
5338
5339 Value *ABLowerShadow = IRB.CreateOr(LHS: ALowerShadow, RHS: BLowerShadow);
5340
5341 Value *WriteThroughLowerShadow = extractLowerShadow(IRB, V: WriteThrough);
5342
5343 Mask = IRB.CreateBitCast(
5344 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: NumElements));
5345 Value *MaskLower =
5346 IRB.CreateExtractElement(Vec: Mask, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5347
5348 Value *AShadow = getShadow(V: A);
5349 Value *DstLowerShadow =
5350 IRB.CreateSelect(C: MaskLower, True: ABLowerShadow, False: WriteThroughLowerShadow);
5351 Value *DstShadow = IRB.CreateInsertElement(
5352 Vec: AShadow, NewElt: DstLowerShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0),
5353 Name: "_msprop");
5354
5355 setShadow(V: &I, SV: DstShadow);
5356 setOriginForNaryOp(I);
5357 }
5358
5359 // Approximately handle AVX Galois Field Affine Transformation
5360 //
5361 // e.g.,
5362 // <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
5363 // <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8>, <32 x i8>, i8)
5364 // <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
5365 // Out A x b
5366 // where A and x are packed matrices, b is a vector,
5367 // Out = A * x + b in GF(2)
5368 //
5369 // Multiplication in GF(2) is equivalent to bitwise AND. However, the matrix
5370 // computation also includes a parity calculation.
5371 //
5372 // For the bitwise AND of bits V1 and V2, the exact shadow is:
5373 // Out_Shadow = (V1_Shadow & V2_Shadow)
5374 // | (V1 & V2_Shadow)
5375 // | (V1_Shadow & V2 )
5376 //
5377 // We approximate the shadow of gf2p8affineqb using:
5378 // Out_Shadow = gf2p8affineqb(x_Shadow, A_shadow, 0)
5379 // | gf2p8affineqb(x, A_shadow, 0)
5380 // | gf2p8affineqb(x_Shadow, A, 0)
5381 // | set1_epi8(b_Shadow)
5382 //
5383 // This approximation has false negatives: if an intermediate dot-product
5384 // contains an even number of 1's, the parity is 0.
5385 // It has no false positives.
5386 void handleAVXGF2P8Affine(IntrinsicInst &I) {
5387 IRBuilder<> IRB(&I);
5388
5389 assert(I.arg_size() == 3);
5390 Value *A = I.getOperand(i_nocapture: 0);
5391 Value *X = I.getOperand(i_nocapture: 1);
5392 Value *B = I.getOperand(i_nocapture: 2);
5393
5394 assert(isFixedIntVector(A));
5395 assert(cast<VectorType>(A->getType())
5396 ->getElementType()
5397 ->getScalarSizeInBits() == 8);
5398
5399 assert(A->getType() == X->getType());
5400
5401 assert(B->getType()->isIntegerTy());
5402 assert(B->getType()->getScalarSizeInBits() == 8);
5403
5404 assert(I.getType() == A->getType());
5405
5406 Value *AShadow = getShadow(V: A);
5407 Value *XShadow = getShadow(V: X);
5408 Value *BZeroShadow = getCleanShadow(V: B);
5409
5410 Value *AShadowXShadow = IRB.CreateIntrinsic(
5411 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {XShadow, AShadow, BZeroShadow});
5412 Value *AShadowX = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5413 Args: {X, AShadow, BZeroShadow});
5414 Value *XShadowA = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5415 Args: {XShadow, A, BZeroShadow});
5416
5417 unsigned NumElements = cast<FixedVectorType>(Val: I.getType())->getNumElements();
5418 Value *BShadow = getShadow(V: B);
5419 Value *BBroadcastShadow = getCleanShadow(V: AShadow);
5420 // There is no LLVM IR intrinsic for _mm512_set1_epi8.
5421 // This loop generates a lot of LLVM IR, which we expect that CodeGen will
5422 // lower appropriately (e.g., VPBROADCASTB).
5423 // Besides, b is often a constant, in which case it is fully initialized.
5424 for (unsigned i = 0; i < NumElements; i++)
5425 BBroadcastShadow = IRB.CreateInsertElement(Vec: BBroadcastShadow, NewElt: BShadow, Idx: i);
5426
5427 setShadow(V: &I, SV: IRB.CreateOr(
5428 Ops: {AShadowXShadow, AShadowX, XShadowA, BBroadcastShadow}));
5429 setOriginForNaryOp(I);
5430 }
5431
5432 // Handle Arm NEON vector load intrinsics (vld*).
5433 //
5434 // The WithLane instructions (ld[234]lane) are similar to:
5435 // call {<4 x i32>, <4 x i32>, <4 x i32>}
5436 // @llvm.aarch64.neon.ld3lane.v4i32.p0
5437 // (<4 x i32> %L1, <4 x i32> %L2, <4 x i32> %L3, i64 %lane, ptr
5438 // %A)
5439 //
5440 // The non-WithLane instructions (ld[234], ld1x[234], ld[234]r) are similar
5441 // to:
5442 // call {<8 x i8>, <8 x i8>} @llvm.aarch64.neon.ld2.v8i8.p0(ptr %A)
5443 void handleNEONVectorLoad(IntrinsicInst &I, bool WithLane) {
5444 unsigned int numArgs = I.arg_size();
5445
5446 // Return type is a struct of vectors of integers or floating-point
5447 assert(I.getType()->isStructTy());
5448 [[maybe_unused]] StructType *RetTy = cast<StructType>(Val: I.getType());
5449 assert(RetTy->getNumElements() > 0);
5450 assert(RetTy->getElementType(0)->isIntOrIntVectorTy() ||
5451 RetTy->getElementType(0)->isFPOrFPVectorTy());
5452 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5453 assert(RetTy->getElementType(i) == RetTy->getElementType(0));
5454
5455 if (WithLane) {
5456 // 2, 3 or 4 vectors, plus lane number, plus input pointer
5457 assert(4 <= numArgs && numArgs <= 6);
5458
5459 // Return type is a struct of the input vectors
5460 assert(RetTy->getNumElements() + 2 == numArgs);
5461 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5462 assert(I.getArgOperand(i)->getType() == RetTy->getElementType(0));
5463 } else {
5464 assert(numArgs == 1);
5465 }
5466
5467 IRBuilder<> IRB(&I);
5468
5469 SmallVector<Value *, 6> ShadowArgs;
5470 if (WithLane) {
5471 for (unsigned int i = 0; i < numArgs - 2; i++)
5472 ShadowArgs.push_back(Elt: getShadow(V: I.getArgOperand(i)));
5473
5474 // Lane number, passed verbatim
5475 Value *LaneNumber = I.getArgOperand(i: numArgs - 2);
5476 ShadowArgs.push_back(Elt: LaneNumber);
5477
5478 // TODO: blend shadow of lane number into output shadow?
5479 insertCheckShadowOf(Val: LaneNumber, OrigIns: &I);
5480 }
5481
5482 Value *Src = I.getArgOperand(i: numArgs - 1);
5483 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
5484
5485 Type *SrcShadowTy = getShadowTy(V: Src);
5486 auto [SrcShadowPtr, SrcOriginPtr] =
5487 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment: Align(1), /*isStore*/ false);
5488 ShadowArgs.push_back(Elt: SrcShadowPtr);
5489
5490 // The NEON vector load instructions handled by this function all have
5491 // integer variants. It is easier to use those rather than trying to cast
5492 // a struct of vectors of floats into a struct of vectors of integers.
5493 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5494 RetTy: getShadowTy(V: &I), ID: I.getIntrinsicID(), Args: ShadowArgs);
5495 setShadow(V: &I, SV: CI);
5496
5497 if (!MS.TrackOrigins)
5498 return;
5499
5500 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
5501 setOrigin(V: &I, Origin: PtrSrcOrigin);
5502 }
5503
5504 /// Handle Arm NEON vector store intrinsics (vst{2,3,4}, vst1x_{2,3,4},
5505 /// and vst{2,3,4}lane).
5506 ///
5507 /// Arm NEON vector store intrinsics have the output address (pointer) as the
5508 /// last argument, with the initial arguments being the inputs (and lane
5509 /// number for vst{2,3,4}lane). They return void.
5510 ///
5511 /// - st4 interleaves the output e.g., st4 (inA, inB, inC, inD, outP) writes
5512 /// abcdabcdabcdabcd... into *outP
5513 /// - st1_x4 is non-interleaved e.g., st1_x4 (inA, inB, inC, inD, outP)
5514 /// writes aaaa...bbbb...cccc...dddd... into *outP
5515 /// - st4lane has arguments of (inA, inB, inC, inD, lane, outP)
5516 /// These instructions can all be instrumented with essentially the same
5517 /// MSan logic, simply by applying the corresponding intrinsic to the shadow.
5518 void handleNEONVectorStoreIntrinsic(IntrinsicInst &I, bool useLane) {
5519 IRBuilder<> IRB(&I);
5520
5521 // Don't use getNumOperands() because it includes the callee
5522 int numArgOperands = I.arg_size();
5523
5524 // The last arg operand is the output (pointer)
5525 assert(numArgOperands >= 1);
5526 Value *Addr = I.getArgOperand(i: numArgOperands - 1);
5527 assert(Addr->getType()->isPointerTy());
5528 int skipTrailingOperands = 1;
5529
5530 if (Opts.msan_check_access_address)
5531 insertCheckShadowOf(Val: Addr, OrigIns: &I);
5532
5533 // Second-last operand is the lane number (for vst{2,3,4}lane)
5534 if (useLane) {
5535 skipTrailingOperands++;
5536 assert(numArgOperands >= static_cast<int>(skipTrailingOperands));
5537 assert(isa<IntegerType>(
5538 I.getArgOperand(numArgOperands - skipTrailingOperands)->getType()));
5539 }
5540
5541 SmallVector<Value *, 8> ShadowArgs;
5542 // All the initial operands are the inputs
5543 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++) {
5544 assert(isa<FixedVectorType>(I.getArgOperand(i)->getType()));
5545 Value *Shadow = getShadow(I: &I, i);
5546 ShadowArgs.append(NumInputs: 1, Elt: Shadow);
5547 }
5548
5549 // MSan's GetShadowTy assumes the LHS is the type we want the shadow for
5550 // e.g., for:
5551 // [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
5552 // we know the type of the output (and its shadow) is <16 x i8>.
5553 //
5554 // Arm NEON VST is unusual because the last argument is the output address:
5555 // define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) {
5556 // call void @llvm.aarch64.neon.st2.v16i8.p0
5557 // (<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
5558 // and we have no type information about P's operand. We must manually
5559 // compute the type (<16 x i8> x 2).
5560 FixedVectorType *OutputVectorTy = FixedVectorType::get(
5561 ElementType: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getElementType(),
5562 NumElts: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements() *
5563 (numArgOperands - skipTrailingOperands));
5564 Type *OutputShadowTy = getShadowTy(OrigTy: OutputVectorTy);
5565
5566 if (useLane)
5567 ShadowArgs.append(NumInputs: 1,
5568 Elt: I.getArgOperand(i: numArgOperands - skipTrailingOperands));
5569
5570 Value *OutputShadowPtr, *OutputOriginPtr;
5571 // AArch64 NEON does not need alignment (unless OS requires it)
5572 std::tie(args&: OutputShadowPtr, args&: OutputOriginPtr) = getShadowOriginPtr(
5573 Addr, IRB, ShadowTy: OutputShadowTy, Alignment: Align(1), /*isStore*/ true);
5574 ShadowArgs.append(NumInputs: 1, Elt: OutputShadowPtr);
5575
5576 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5577 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
5578 setShadow(V: &I, SV: CI);
5579
5580 if (MS.TrackOrigins) {
5581 // TODO: if we modelled the vst* instruction more precisely, we could
5582 // more accurately track the origins (e.g., if both inputs are
5583 // uninitialized for vst2, we currently blame the second input, even
5584 // though part of the output depends only on the first input).
5585 //
5586 // This is particularly imprecise for vst{2,3,4}lane, since only one
5587 // lane of each input is actually copied to the output.
5588 OriginCombiner OC(this, IRB);
5589 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++)
5590 OC.Add(V: I.getArgOperand(i));
5591
5592 const DataLayout &DL = F.getDataLayout();
5593 OC.DoneAndStoreOrigin(TS: DL.getTypeStoreSize(Ty: OutputVectorTy),
5594 OriginPtr: OutputOriginPtr);
5595 }
5596 }
5597
5598 // Integer matrix multiplication:
5599 // - <4 x i32> @llvm.aarch64.neon.{s,u,us}mmla.v4i32.v16i8
5600 // (<4 x i32> %R, <16 x i8> %A, <16 x i8> %B)
5601 // - <4 x i32> is a 2x2 matrix
5602 // - <16 x i8> %A and %B are 2x8 and 8x2 matrices respectively
5603 //
5604 // Floating-point matrix multiplication:
5605 // - <4 x float> @llvm.aarch64.neon.bfmmla
5606 // (<4 x float> %R, <8 x bfloat> %A, <8 x bfloat> %B)
5607 // - <4 x float> is a 2x2 matrix
5608 // - <8 x bfloat> %A and %B are 2x4 and 4x2 matrices respectively
5609 //
5610 // The general shadow propagation approach is:
5611 // 1) get the shadows of the input matrices %A and %B
5612 // 2) map each shadow value to 0x1 if the corresponding value is fully
5613 // initialized, and 0x0 otherwise
5614 // 3) perform a matrix multiplication on the shadows of %A and %B [*].
5615 // The output will be a 2x2 matrix. For each element, a value of 0x8
5616 // (for {s,u,us}mmla) or 0x4 (for bfmmla) means all the corresponding
5617 // inputs were clean; if so, set the shadow to zero, otherwise set to -1.
5618 // 4) blend in the shadow of %R
5619 //
5620 // [*] Since shadows are integral, the obvious approach is to always apply
5621 // ummla to the shadows. Unfortunately, Armv8.2+bf16 supports bfmmla,
5622 // but not ummla. Thus, for bfmmla, our instrumentation reuses bfmmla.
5623 //
5624 // TODO: consider allowing multiplication of zero with an uninitialized value
5625 // to result in an initialized value.
5626 void handleNEONMatrixMultiply(IntrinsicInst &I) {
5627 IRBuilder<> IRB(&I);
5628
5629 assert(I.arg_size() == 3);
5630 Value *R = I.getArgOperand(i: 0);
5631 Value *A = I.getArgOperand(i: 1);
5632 Value *B = I.getArgOperand(i: 2);
5633
5634 assert(I.getType() == R->getType());
5635
5636 assert(isa<FixedVectorType>(R->getType()));
5637 assert(isa<FixedVectorType>(A->getType()));
5638 assert(isa<FixedVectorType>(B->getType()));
5639
5640 FixedVectorType *RTy = cast<FixedVectorType>(Val: R->getType());
5641 FixedVectorType *ATy = cast<FixedVectorType>(Val: A->getType());
5642 FixedVectorType *BTy = cast<FixedVectorType>(Val: B->getType());
5643 assert(ATy->getElementType() == BTy->getElementType());
5644
5645 if (RTy->getElementType()->isIntegerTy()) {
5646 // <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8
5647 // (<4 x i32> %R, <16 x i8> %X, <16 x i8> %Y)
5648 assert(RTy == FixedVectorType::get(IntegerType::get(*MS.C, 32), 4));
5649 assert(ATy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5650 assert(BTy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5651 } else {
5652 // <4 x float> @llvm.aarch64.neon.bfmmla
5653 // (<4 x float> %R, <8 x bfloat> %X, <8 x bfloat> %Y)
5654 assert(RTy == FixedVectorType::get(Type::getFloatTy(*MS.C), 4));
5655 assert(ATy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5656 assert(BTy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5657 }
5658
5659 Value *ShadowR = getShadow(I: &I, i: 0);
5660 Value *ShadowA = getShadow(I: &I, i: 1);
5661 Value *ShadowB = getShadow(I: &I, i: 2);
5662
5663 Value *ShadowAB;
5664 Value *FullyInit;
5665
5666 if (RTy->getElementType()->isIntegerTy()) {
5667 // If the value is fully initialized, the shadow will be 000...001.
5668 // Otherwise, the shadow will be all zero.
5669 // (This is the opposite of how we typically handle shadows.)
5670 ShadowA = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5671 DestTy: getShadowTy(OrigTy: ATy));
5672 ShadowB = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5673 DestTy: getShadowTy(OrigTy: BTy));
5674 // TODO: the CreateSelect approach used below for floating-point is more
5675 // generic than CreateZExt. Investigate whether it is worthwhile
5676 // unifying the two approaches.
5677
5678 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_ummla,
5679 Args: {getCleanShadow(OrigTy: RTy), ShadowA, ShadowB});
5680
5681 // ummla multiplies a 2x8 matrix with an 8x2 matrix. If all entries of the
5682 // input matrices are equal to 0x1, all entries of the output matrix will
5683 // be 0x8.
5684 FullyInit = ConstantVector::getSplat(
5685 EC: RTy->getElementCount(), Elt: ConstantInt::get(Ty: RTy->getElementType(), V: 0x8));
5686
5687 ShadowAB = IRB.CreateICmpNE(LHS: ShadowAB, RHS: FullyInit);
5688 } else {
5689 Constant *ABZeros = ConstantVector::getSplat(
5690 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 0));
5691 Constant *ABOnes = ConstantVector::getSplat(
5692 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 1));
5693
5694 // As per the integer case, if the shadow is clean, we store 0x1,
5695 // otherwise we store 0x0 (the opposite of usual shadow arithmetic).
5696 ShadowA = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5697 True: ABOnes, False: ABZeros);
5698 ShadowB = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5699 True: ABOnes, False: ABZeros);
5700
5701 Constant *RZeros = ConstantVector::getSplat(
5702 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 0));
5703
5704 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_bfmmla,
5705 Args: {RZeros, ShadowA, ShadowB});
5706
5707 // bfmmla multiplies a 2x4 matrix with an 4x2 matrix. If all entries of
5708 // the input matrices are equal to 0x1, all entries of the output matrix
5709 // will be 4.0. (To avoid floating-point error, we check if each entry
5710 // < 3.5.)
5711 FullyInit = ConstantVector::getSplat(
5712 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 3.5));
5713
5714 // FCmpULT: "yields true if either operand is a QNAN or op1 is less than"
5715 // op2"
5716 ShadowAB = IRB.CreateFCmpULT(LHS: ShadowAB, RHS: FullyInit);
5717 }
5718
5719 ShadowR = IRB.CreateICmpNE(LHS: ShadowR, RHS: getCleanShadow(OrigTy: RTy));
5720 ShadowR = IRB.CreateOr(LHS: ShadowAB, RHS: ShadowR);
5721
5722 setShadow(V: &I, SV: IRB.CreateSExt(V: ShadowR, DestTy: getShadowTy(OrigTy: RTy)));
5723
5724 setOriginForNaryOp(I);
5725 }
5726
5727 /// Handle intrinsics by applying the intrinsic to the shadows.
5728 ///
5729 /// For example, this can be applied to the Arm NEON vector table intrinsics
5730 /// (tbl{1,2,3,4}).
5731 ///
5732 /// Typically, shadowIntrinsicID will be specified by the caller to be
5733 /// I.getIntrinsicID(), but the caller can choose to replace it with another
5734 /// intrinsic of the same type.
5735 ///
5736 /// The trailing arguments are passed verbatim to the intrinsic, though any
5737 /// uninitialized trailing arguments can also taint the shadow e.g., for an
5738 /// intrinsic with one trailing verbatim argument:
5739 /// out = intrinsic(var1, var2, opType)
5740 /// we compute:
5741 /// shadow[out] =
5742 /// intrinsic(shadow[var1], shadow[var2], opType) | shadow[opType]
5743 ///
5744 /// If an intrinsic is called with floating-point arguments, we will
5745 /// typically cast the shadows to floating-point, apply the intrinsic [*],
5746 /// then cast the result back to integer/shadow.
5747 ///
5748 /// In cases where we know the intrinsic is compatible with integer
5749 /// arguments, 'forceIntegerIntrinsic' will apply the integer variant, even
5750 /// if the arguments are floating-point, thus avoiding unnecessary casts
5751 /// e.g., if I is:
5752 /// <16 x float> @llvm.x86.avx512.mask.compress
5753 /// (<16 x float>, <16 x float>, <16 x i1> %mask)
5754 /// we would prefer to compute the shadows using:
5755 /// <16 x i32> @llvm.x86.avx512.mask.compress
5756 /// (<16 x i32>, <16 x i32>, <16 x i1> %mask)
5757 ///
5758 /// [*] CAUTION: this assumes that the intrinsic will handle arbitrary
5759 /// bit-patterns (for example, if the intrinsic accepts floats
5760 /// for var1, we require that it doesn't care if inputs are
5761 /// NaNs).
5762 ///
5763 /// The origin is approximated using setOriginForNaryOp.
5764 void handleIntrinsicByApplyingToShadow(IntrinsicInst &I,
5765 Intrinsic::ID shadowIntrinsicID,
5766 unsigned int trailingVerbatimArgs,
5767 bool forceIntegerIntrinsic) {
5768 IRBuilder<> IRB(&I);
5769
5770 assert(trailingVerbatimArgs < I.arg_size());
5771
5772 SmallVector<Value *, 8> ShadowArgs;
5773 // Don't use getNumOperands() because it includes the callee
5774 for (unsigned int i = 0; i < I.arg_size() - trailingVerbatimArgs; i++) {
5775 Value *Shadow = getShadow(I: &I, i);
5776
5777 if (forceIntegerIntrinsic)
5778 ShadowArgs.push_back(Elt: Shadow);
5779 else
5780 ShadowArgs.push_back(
5781 Elt: IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i)->getType()));
5782 }
5783
5784 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5785 i++) {
5786 Value *Arg = I.getArgOperand(i);
5787 if (forceIntegerIntrinsic)
5788 assert(Arg->getType()->isIntOrIntVectorTy());
5789 ShadowArgs.push_back(Elt: Arg);
5790 }
5791
5792 Value *CombinedShadow;
5793 if (forceIntegerIntrinsic) {
5794 CombinedShadow =
5795 IRB.CreateIntrinsic(RetTy: getShadowTy(V: &I), ID: shadowIntrinsicID, Args: ShadowArgs);
5796 } else {
5797 Value *CI =
5798 IRB.CreateIntrinsic(RetTy: I.getType(), ID: shadowIntrinsicID, Args: ShadowArgs);
5799 CombinedShadow = IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I));
5800 }
5801
5802 // Combine the computed shadow with the shadow of trailing args
5803 for (unsigned int i = I.arg_size() - trailingVerbatimArgs; i < I.arg_size();
5804 i++) {
5805 Value *Shadow =
5806 CreateShadowCast(IRB, V: getShadow(I: &I, i), dstTy: CombinedShadow->getType());
5807 CombinedShadow = IRB.CreateOr(LHS: Shadow, RHS: CombinedShadow, Name: "_msprop");
5808 }
5809
5810 setShadow(V: &I, SV: CombinedShadow);
5811
5812 setOriginForNaryOp(I);
5813 }
5814
5815 // Approximation only
5816 //
5817 // e.g., <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
5818 void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
5819 assert(I.arg_size() == 2);
5820
5821 handleShadowOr(I);
5822 }
5823
5824 // Handles:
5825 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane
5826 // (<4 x half>, <8 x i8>, <16 x i8>, i32)
5827 // accumulator A B lane
5828 //
5829 // <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane
5830 // (<8 x half>, <16 x i8>, <16 x i8>, i32)
5831 // <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane
5832 // (<2 x float>, <8 x i8>, <16 x i8>, i32)
5833 // <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane
5834 // (<4 x float>, <16 x i8>, <16 x i8>, i32)
5835 //
5836 // The lane specifies which pair (fdot2) or quad (fdot4) of numbers to
5837 // extract from B, which is then splatted before being used in the dot
5838 // products e.g., for
5839 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane:
5840 // (<4 x half>, <8 x i8>, <16 x i8>, 1)
5841 //
5842 // acc[0] acc[1] acc[2] acc[3]
5843 // + + + + + + + +
5844 // A[0] A[1] A[2] A[3] A[4] A[5] A[6] A[7]
5845 // * * * * * * * *
5846 // B[2] B[3] B[2] B[3] B[2] B[3] B[2] B[3]
5847 //
5848 // Notice that if any bit of B[2] or B[3] is uninitialized, every accumulator
5849 // value will become tainted; we approximate this by marking the output as
5850 // fully uninitialized. This permits a 'Select' optimization.
5851 //
5852 // This function is separate from handleVectorDotProductIntrinsic(), because
5853 // the non-overlapping features (e.g., odd/even lanes vs. numbered lanes,
5854 // ZeroPurifies, EltSizeInBits) and optimizations make clean code reuse
5855 // difficult.
5856 void handleNEONDotProductLaneIntrinsic(IntrinsicInst &I,
5857 unsigned ReductionFactor) {
5858 IRBuilder<> IRB(&I);
5859 assert(I.arg_size() == 4);
5860
5861 [[maybe_unused]] Value *VAcc = I.getOperand(i_nocapture: 0);
5862 [[maybe_unused]] Value *Va = I.getOperand(i_nocapture: 1);
5863 [[maybe_unused]] Value *Vb = I.getOperand(i_nocapture: 2);
5864 Value *Lane = I.getOperand(i_nocapture: 3);
5865
5866 assert(isa<FixedVectorType>(VAcc->getType()));
5867 assert(VAcc->getType() == I.getType());
5868
5869 assert(isa<FixedVectorType>(Va->getType()));
5870 assert(Va->getType()->getPrimitiveSizeInBits() ==
5871 I.getType()->getPrimitiveSizeInBits());
5872
5873 assert(cast<FixedVectorType>(Va->getType())->getNumElements() ==
5874 cast<FixedVectorType>(I.getType())->getNumElements() *
5875 ReductionFactor);
5876
5877 assert(isa<FixedVectorType>(Vb->getType()));
5878 // Deliberately not strict equality
5879 assert(Vb->getType()->getPrimitiveSizeInBits() >=
5880 I.getType()->getPrimitiveSizeInBits());
5881
5882 assert(Lane->getType()->isIntegerTy());
5883
5884 // (<4 x 16>, <8 x i8>, <16 x i8>)
5885 // SAcc Sa Sb
5886 Value *SAcc = getShadow(I: &I, i: 0);
5887 Value *Sa = getShadow(I: &I, i: 1);
5888 Value *Sb = getShadow(I: &I, i: 2);
5889
5890 // Cast the shadows to:
5891 // (<4 x i16>, <4 x i16>, <8 x i16>)
5892 // SAcc Sa Sb
5893 Sa = IRB.CreateBitCast(V: Sa, DestTy: SAcc->getType());
5894 Sb = IRB.CreateBitCast(
5895 V: Sb, DestTy: FixedVectorType::getWithSizeAndScalar(
5896 SizeTy: cast<FixedVectorType>(Val: Sb->getType()),
5897 EltTy: cast<FixedVectorType>(Val: SAcc->getType())->getElementType()));
5898
5899 // All-or-nothing shadows
5900 Sa =
5901 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: Sa, RHS: getCleanShadow(V: Sa)), DestTy: Sa->getType());
5902
5903 // Extract the specific lane from Sb to get i16, then turn it into a single
5904 // bit representing if it is fully initialized.
5905 Sb = IRB.CreateExtractElement(Vec: Sb, Idx: Lane);
5906 Value *SbClean = IRB.CreateIsNull(Arg: Sb);
5907
5908 Value *SOutput = IRB.CreateOr(LHS: SAcc, RHS: Sa);
5909
5910 // Select is cheaper than broadcasting Sb into <4 x i16>.
5911 SOutput = IRB.CreateSelect(C: SbClean, True: SOutput, False: getPoisonedShadow(V: SOutput));
5912
5913 setShadow(V: &I, SV: SOutput);
5914 setOriginForNaryOp(I);
5915 }
5916
5917 bool maybeHandleCrossPlatformIntrinsic(IntrinsicInst &I) {
5918 switch (I.getIntrinsicID()) {
5919 case Intrinsic::uadd_with_overflow:
5920 case Intrinsic::sadd_with_overflow:
5921 case Intrinsic::usub_with_overflow:
5922 case Intrinsic::ssub_with_overflow:
5923 case Intrinsic::umul_with_overflow:
5924 case Intrinsic::smul_with_overflow:
5925 handleArithmeticWithOverflow(I);
5926 break;
5927 case Intrinsic::modf:
5928 case Intrinsic::sincos:
5929 case Intrinsic::sincospi:
5930 handleModfOrSincos(I);
5931 break;
5932 case Intrinsic::abs:
5933 handleAbsIntrinsic(I);
5934 break;
5935 case Intrinsic::bitreverse:
5936 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
5937 /*trailingVerbatimArgs=*/0,
5938 /*forceIntegerIntrinsic=*/false);
5939 break;
5940 case Intrinsic::is_fpclass:
5941 handleIsFpClass(I);
5942 break;
5943 case Intrinsic::lifetime_start:
5944 handleLifetimeStart(I);
5945 break;
5946 case Intrinsic::launder_invariant_group:
5947 handleInvariantGroup(I);
5948 break;
5949 case Intrinsic::bswap:
5950 handleBswap(I);
5951 break;
5952 case Intrinsic::ctlz:
5953 case Intrinsic::cttz:
5954 handleCountLeadingTrailingZeros(I);
5955 break;
5956 case Intrinsic::masked_compressstore:
5957 handleMaskedCompressStore(I);
5958 break;
5959 case Intrinsic::masked_expandload:
5960 handleMaskedExpandLoad(I);
5961 break;
5962 case Intrinsic::masked_gather:
5963 handleMaskedGather(I);
5964 break;
5965 case Intrinsic::masked_scatter:
5966 handleMaskedScatter(I);
5967 break;
5968 case Intrinsic::masked_store:
5969 handleMaskedStore(I);
5970 break;
5971 case Intrinsic::masked_load:
5972 handleMaskedLoad(I);
5973 break;
5974 case Intrinsic::masked_udiv:
5975 case Intrinsic::masked_sdiv:
5976 case Intrinsic::masked_urem:
5977 case Intrinsic::masked_srem:
5978 handleMaskedIntegerDivRem(I);
5979 break;
5980 case Intrinsic::vector_reduce_and:
5981 handleVectorReduceAndIntrinsic(I);
5982 break;
5983 case Intrinsic::vector_reduce_or:
5984 handleVectorReduceOrIntrinsic(I);
5985 break;
5986
5987 case Intrinsic::vector_reduce_add:
5988 case Intrinsic::vector_reduce_xor:
5989 case Intrinsic::vector_reduce_mul:
5990 // Signed/Unsigned Min/Max
5991 // TODO: handling similarly to AND/OR may be more precise.
5992 case Intrinsic::vector_reduce_smax:
5993 case Intrinsic::vector_reduce_smin:
5994 case Intrinsic::vector_reduce_umax:
5995 case Intrinsic::vector_reduce_umin:
5996 // TODO: this has no false positives, but arguably we should check that all
5997 // the bits are initialized.
5998 case Intrinsic::vector_reduce_fmax:
5999 case Intrinsic::vector_reduce_fmin:
6000 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/false);
6001 break;
6002
6003 case Intrinsic::vector_reduce_fadd:
6004 case Intrinsic::vector_reduce_fmul:
6005 handleVectorReduceWithStarterIntrinsic(I);
6006 break;
6007
6008 // e.g., <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32>, <8 x i32>)
6009 case Intrinsic::vector_partial_reduce_add:
6010 case Intrinsic::vector_partial_reduce_fadd:
6011 handleVectorPartialReduceAddIntrinsic(I);
6012 break;
6013
6014 case Intrinsic::scmp:
6015 case Intrinsic::ucmp: {
6016 handleShadowOr(I);
6017 break;
6018 }
6019
6020 case Intrinsic::fshl:
6021 case Intrinsic::fshr:
6022 handleFunnelShift(I);
6023 break;
6024
6025 case Intrinsic::pdep:
6026 case Intrinsic::pext:
6027 handleGenericBitManipulation(I);
6028 break;
6029
6030 case Intrinsic::is_constant:
6031 // The result of llvm.is.constant() is always defined.
6032 setShadow(V: &I, SV: getCleanShadow(V: &I));
6033 setOrigin(V: &I, Origin: getCleanOrigin());
6034 break;
6035
6036 // The non-saturating versions are handled by visitFPTo[US]IInst().
6037 //
6038 // N.B. some platform-specific intrinsics, such as AArch64 fcvtz[us], are
6039 // lowered to these cross-platform intrinsics.
6040 case Intrinsic::fptosi_sat:
6041 case Intrinsic::fptoui_sat:
6042 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
6043 break;
6044
6045 // e.g.,
6046 // notail call void (...) @llvm.fake.use(i64 %x)
6047 // notail call void (...) @llvm.fake.use(i32 %y)
6048 // notail call void (...) @llvm.fake.use(ptr %z)
6049 case Intrinsic::fake_use:
6050 assert(I.getType()->isVoidTy());
6051 // fake_uses aren't real, they can't hurt you. If the use isn't real, it
6052 // can't be a real use-of-uninitialized memory. Silently skip over
6053 // fake_use.
6054 return true;
6055
6056 default:
6057 return false;
6058 }
6059
6060 return true;
6061 }
6062
6063 bool maybeHandleX86SIMDIntrinsic(IntrinsicInst &I) {
6064 switch (I.getIntrinsicID()) {
6065 case Intrinsic::x86_sse_stmxcsr:
6066 handleStmxcsr(I);
6067 break;
6068 case Intrinsic::x86_sse_ldmxcsr:
6069 handleLdmxcsr(I);
6070 break;
6071
6072 // Convert Scalar Double Precision Floating-Point Value
6073 // to Unsigned Doubleword Integer
6074 // etc.
6075 case Intrinsic::x86_avx512_vcvtsd2usi64:
6076 case Intrinsic::x86_avx512_vcvtsd2usi32:
6077 case Intrinsic::x86_avx512_vcvtss2usi64:
6078 case Intrinsic::x86_avx512_vcvtss2usi32:
6079 case Intrinsic::x86_avx512_cvttss2usi64:
6080 case Intrinsic::x86_avx512_cvttss2usi:
6081 case Intrinsic::x86_avx512_cvttsd2usi64:
6082 case Intrinsic::x86_avx512_cvttsd2usi:
6083 case Intrinsic::x86_avx512_cvtusi2ss:
6084 case Intrinsic::x86_avx512_cvtusi642sd:
6085 case Intrinsic::x86_avx512_cvtusi642ss:
6086 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1, HasRoundingMode: true);
6087 break;
6088 case Intrinsic::x86_sse2_cvtsd2si64:
6089 case Intrinsic::x86_sse2_cvtsd2si:
6090 case Intrinsic::x86_sse2_cvtsd2ss:
6091 case Intrinsic::x86_sse2_cvttsd2si64:
6092 case Intrinsic::x86_sse2_cvttsd2si:
6093 case Intrinsic::x86_sse_cvtss2si64:
6094 case Intrinsic::x86_sse_cvtss2si:
6095 case Intrinsic::x86_sse_cvttss2si64:
6096 case Intrinsic::x86_sse_cvttss2si:
6097 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1);
6098 break;
6099 case Intrinsic::x86_sse_cvtps2pi:
6100 case Intrinsic::x86_sse_cvttps2pi:
6101 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 2);
6102 break;
6103
6104 // TODO:
6105 // <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double>)
6106 // <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64>)
6107 // <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, <1 x i64>)
6108
6109 case Intrinsic::x86_vcvtps2ph_128:
6110 case Intrinsic::x86_vcvtps2ph_256: {
6111 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/true);
6112 break;
6113 }
6114
6115 // Convert Packed Single Precision Floating-Point Values
6116 // to Packed Signed Doubleword Integer Values
6117 //
6118 // <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
6119 // (<16 x float>, <16 x i32>, i16, i32)
6120 case Intrinsic::x86_avx512_mask_cvtps2dq_512:
6121 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/false);
6122 break;
6123
6124 // Convert Packed Double Precision Floating-Point Values
6125 // to Packed Single Precision Floating-Point Values
6126 case Intrinsic::x86_sse2_cvtpd2ps:
6127 case Intrinsic::x86_sse2_cvtps2dq:
6128 case Intrinsic::x86_sse2_cvtpd2dq:
6129 case Intrinsic::x86_sse2_cvttps2dq:
6130 case Intrinsic::x86_sse2_cvttpd2dq:
6131 case Intrinsic::x86_avx_cvt_pd2_ps_256:
6132 case Intrinsic::x86_avx_cvt_ps2dq_256:
6133 case Intrinsic::x86_avx_cvt_pd2dq_256:
6134 case Intrinsic::x86_avx_cvtt_ps2dq_256:
6135 case Intrinsic::x86_avx_cvtt_pd2dq_256: {
6136 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/false);
6137 break;
6138 }
6139
6140 // Convert Single-Precision FP Value to 16-bit FP Value
6141 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
6142 // (<16 x float>, i32, <16 x i16>, i16)
6143 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
6144 // (<4 x float>, i32, <8 x i16>, i8)
6145 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256
6146 // (<8 x float>, i32, <8 x i16>, i8)
6147 case Intrinsic::x86_avx512_mask_vcvtps2ph_512:
6148 case Intrinsic::x86_avx512_mask_vcvtps2ph_256:
6149 case Intrinsic::x86_avx512_mask_vcvtps2ph_128:
6150 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/true);
6151 break;
6152
6153 // Shift Packed Data (Left Logical, Right Arithmetic, Right Logical)
6154 case Intrinsic::x86_avx512_psll_w_512:
6155 case Intrinsic::x86_avx512_psll_d_512:
6156 case Intrinsic::x86_avx512_psll_q_512:
6157 case Intrinsic::x86_avx512_pslli_w_512:
6158 case Intrinsic::x86_avx512_pslli_d_512:
6159 case Intrinsic::x86_avx512_pslli_q_512:
6160 case Intrinsic::x86_avx512_psrl_w_512:
6161 case Intrinsic::x86_avx512_psrl_d_512:
6162 case Intrinsic::x86_avx512_psrl_q_512:
6163 case Intrinsic::x86_avx512_psra_w_512:
6164 case Intrinsic::x86_avx512_psra_d_512:
6165 case Intrinsic::x86_avx512_psra_q_512:
6166 case Intrinsic::x86_avx512_psrli_w_512:
6167 case Intrinsic::x86_avx512_psrli_d_512:
6168 case Intrinsic::x86_avx512_psrli_q_512:
6169 case Intrinsic::x86_avx512_psrai_w_512:
6170 case Intrinsic::x86_avx512_psrai_d_512:
6171 case Intrinsic::x86_avx512_psrai_q_512:
6172 case Intrinsic::x86_avx512_psra_q_256:
6173 case Intrinsic::x86_avx512_psra_q_128:
6174 case Intrinsic::x86_avx512_psrai_q_256:
6175 case Intrinsic::x86_avx512_psrai_q_128:
6176 case Intrinsic::x86_avx2_psll_w:
6177 case Intrinsic::x86_avx2_psll_d:
6178 case Intrinsic::x86_avx2_psll_q:
6179 case Intrinsic::x86_avx2_pslli_w:
6180 case Intrinsic::x86_avx2_pslli_d:
6181 case Intrinsic::x86_avx2_pslli_q:
6182 case Intrinsic::x86_avx2_psrl_w:
6183 case Intrinsic::x86_avx2_psrl_d:
6184 case Intrinsic::x86_avx2_psrl_q:
6185 case Intrinsic::x86_avx2_psra_w:
6186 case Intrinsic::x86_avx2_psra_d:
6187 case Intrinsic::x86_avx2_psrli_w:
6188 case Intrinsic::x86_avx2_psrli_d:
6189 case Intrinsic::x86_avx2_psrli_q:
6190 case Intrinsic::x86_avx2_psrai_w:
6191 case Intrinsic::x86_avx2_psrai_d:
6192 case Intrinsic::x86_sse2_psll_w:
6193 case Intrinsic::x86_sse2_psll_d:
6194 case Intrinsic::x86_sse2_psll_q:
6195 case Intrinsic::x86_sse2_pslli_w:
6196 case Intrinsic::x86_sse2_pslli_d:
6197 case Intrinsic::x86_sse2_pslli_q:
6198 case Intrinsic::x86_sse2_psrl_w:
6199 case Intrinsic::x86_sse2_psrl_d:
6200 case Intrinsic::x86_sse2_psrl_q:
6201 case Intrinsic::x86_sse2_psra_w:
6202 case Intrinsic::x86_sse2_psra_d:
6203 case Intrinsic::x86_sse2_psrli_w:
6204 case Intrinsic::x86_sse2_psrli_d:
6205 case Intrinsic::x86_sse2_psrli_q:
6206 case Intrinsic::x86_sse2_psrai_w:
6207 case Intrinsic::x86_sse2_psrai_d:
6208 case Intrinsic::x86_mmx_psll_w:
6209 case Intrinsic::x86_mmx_psll_d:
6210 case Intrinsic::x86_mmx_psll_q:
6211 case Intrinsic::x86_mmx_pslli_w:
6212 case Intrinsic::x86_mmx_pslli_d:
6213 case Intrinsic::x86_mmx_pslli_q:
6214 case Intrinsic::x86_mmx_psrl_w:
6215 case Intrinsic::x86_mmx_psrl_d:
6216 case Intrinsic::x86_mmx_psrl_q:
6217 case Intrinsic::x86_mmx_psra_w:
6218 case Intrinsic::x86_mmx_psra_d:
6219 case Intrinsic::x86_mmx_psrli_w:
6220 case Intrinsic::x86_mmx_psrli_d:
6221 case Intrinsic::x86_mmx_psrli_q:
6222 case Intrinsic::x86_mmx_psrai_w:
6223 case Intrinsic::x86_mmx_psrai_d:
6224 handleVectorShiftIntrinsic(I, /* Variable */ false);
6225 break;
6226 case Intrinsic::x86_avx2_psllv_d:
6227 case Intrinsic::x86_avx2_psllv_d_256:
6228 case Intrinsic::x86_avx512_psllv_d_512:
6229 case Intrinsic::x86_avx2_psllv_q:
6230 case Intrinsic::x86_avx2_psllv_q_256:
6231 case Intrinsic::x86_avx512_psllv_q_512:
6232 case Intrinsic::x86_avx2_psrlv_d:
6233 case Intrinsic::x86_avx2_psrlv_d_256:
6234 case Intrinsic::x86_avx512_psrlv_d_512:
6235 case Intrinsic::x86_avx2_psrlv_q:
6236 case Intrinsic::x86_avx2_psrlv_q_256:
6237 case Intrinsic::x86_avx512_psrlv_q_512:
6238 case Intrinsic::x86_avx2_psrav_d:
6239 case Intrinsic::x86_avx2_psrav_d_256:
6240 case Intrinsic::x86_avx512_psrav_d_512:
6241 case Intrinsic::x86_avx512_psrav_q_128:
6242 case Intrinsic::x86_avx512_psrav_q_256:
6243 case Intrinsic::x86_avx512_psrav_q_512:
6244 handleVectorShiftIntrinsic(I, /* Variable */ true);
6245 break;
6246
6247 // Pack with Signed/Unsigned Saturation
6248 case Intrinsic::x86_sse2_packsswb_128:
6249 case Intrinsic::x86_sse2_packssdw_128:
6250 case Intrinsic::x86_sse2_packuswb_128:
6251 case Intrinsic::x86_sse41_packusdw:
6252 case Intrinsic::x86_avx2_packsswb:
6253 case Intrinsic::x86_avx2_packssdw:
6254 case Intrinsic::x86_avx2_packuswb:
6255 case Intrinsic::x86_avx2_packusdw:
6256 // e.g., <64 x i8> @llvm.x86.avx512.packsswb.512
6257 // (<32 x i16> %a, <32 x i16> %b)
6258 // <32 x i16> @llvm.x86.avx512.packssdw.512
6259 // (<16 x i32> %a, <16 x i32> %b)
6260 // Note: AVX512 masked variants are auto-upgraded by LLVM.
6261 case Intrinsic::x86_avx512_packsswb_512:
6262 case Intrinsic::x86_avx512_packssdw_512:
6263 case Intrinsic::x86_avx512_packuswb_512:
6264 case Intrinsic::x86_avx512_packusdw_512:
6265 handleVectorPackIntrinsic(I);
6266 break;
6267
6268 case Intrinsic::x86_sse41_pblendvb:
6269 case Intrinsic::x86_sse41_blendvpd:
6270 case Intrinsic::x86_sse41_blendvps:
6271 case Intrinsic::x86_avx_blendv_pd_256:
6272 case Intrinsic::x86_avx_blendv_ps_256:
6273 case Intrinsic::x86_avx2_pblendvb:
6274 handleBlendvIntrinsic(I);
6275 break;
6276
6277 case Intrinsic::x86_avx_dp_ps_256:
6278 case Intrinsic::x86_sse41_dppd:
6279 case Intrinsic::x86_sse41_dpps:
6280 handleDppIntrinsic(I);
6281 break;
6282
6283 case Intrinsic::x86_mmx_packsswb:
6284 case Intrinsic::x86_mmx_packuswb:
6285 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 16);
6286 break;
6287
6288 case Intrinsic::x86_mmx_packssdw:
6289 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 32);
6290 break;
6291
6292 case Intrinsic::x86_mmx_psad_bw:
6293 handleVectorSadIntrinsic(I, IsMMX: true);
6294 break;
6295 case Intrinsic::x86_sse2_psad_bw:
6296 case Intrinsic::x86_avx2_psad_bw:
6297 handleVectorSadIntrinsic(I);
6298 break;
6299
6300 // Multiply and Add Packed Words
6301 // < 4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
6302 // < 8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
6303 // <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
6304 //
6305 // Multiply and Add Packed Signed and Unsigned Bytes
6306 // < 8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
6307 // <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
6308 // <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
6309 //
6310 // These intrinsics are auto-upgraded into non-masked forms:
6311 // < 4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128
6312 // (<8 x i16>, <8 x i16>, <4 x i32>, i8)
6313 // < 8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256
6314 // (<16 x i16>, <16 x i16>, <8 x i32>, i8)
6315 // <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512
6316 // (<32 x i16>, <32 x i16>, <16 x i32>, i16)
6317 // < 8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128
6318 // (<16 x i8>, <16 x i8>, <8 x i16>, i8)
6319 // <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256
6320 // (<32 x i8>, <32 x i8>, <16 x i16>, i16)
6321 // <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512
6322 // (<64 x i8>, <64 x i8>, <32 x i16>, i32)
6323 case Intrinsic::x86_sse2_pmadd_wd:
6324 case Intrinsic::x86_avx2_pmadd_wd:
6325 case Intrinsic::x86_avx512_pmaddw_d_512:
6326 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
6327 case Intrinsic::x86_avx2_pmadd_ub_sw:
6328 case Intrinsic::x86_avx512_pmaddubs_w_512:
6329 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6330 /*ZeroPurifies=*/true,
6331 /*EltSizeInBits=*/0,
6332 /*Lanes=*/kBothLanes);
6333 break;
6334
6335 // <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>)
6336 case Intrinsic::x86_ssse3_pmadd_ub_sw:
6337 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6338 /*ZeroPurifies=*/true,
6339 /*EltSizeInBits=*/8,
6340 /*Lanes=*/kBothLanes);
6341 break;
6342
6343 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>)
6344 case Intrinsic::x86_mmx_pmadd_wd:
6345 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6346 /*ZeroPurifies=*/true,
6347 /*EltSizeInBits=*/16,
6348 /*Lanes=*/kBothLanes);
6349 break;
6350
6351 // BFloat16 multiply-add to single-precision
6352 // <4 x float> llvm.aarch64.neon.bfmlalt
6353 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6354 case Intrinsic::aarch64_neon_bfmlalt:
6355 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6356 /*ZeroPurifies=*/false,
6357 /*EltSizeInBits=*/0,
6358 /*Lanes=*/kOddLanes);
6359 break;
6360
6361 // <4 x float> llvm.aarch64.neon.bfmlalb
6362 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6363 case Intrinsic::aarch64_neon_bfmlalb:
6364 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6365 /*ZeroPurifies=*/false,
6366 /*EltSizeInBits=*/0,
6367 /*Lanes=*/kEvenLanes);
6368 break;
6369
6370 // AVX Vector Neural Network Instructions: bytes
6371 //
6372 // Multiply and Add Signed Bytes
6373 // < 4 x i32> @llvm.x86.avx2.vpdpbssd.128
6374 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6375 // < 8 x i32> @llvm.x86.avx2.vpdpbssd.256
6376 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6377 // <16 x i32> @llvm.x86.avx10.vpdpbssd.512
6378 // (<16 x i32>, <64 x i8>, <64 x i8>)
6379 //
6380 // Multiply and Add Signed Bytes With Saturation
6381 // < 4 x i32> @llvm.x86.avx2.vpdpbssds.128
6382 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6383 // < 8 x i32> @llvm.x86.avx2.vpdpbssds.256
6384 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6385 // <16 x i32> @llvm.x86.avx10.vpdpbssds.512
6386 // (<16 x i32>, <64 x i8>, <64 x i8>)
6387 //
6388 // Multiply and Add Signed and Unsigned Bytes
6389 // < 4 x i32> @llvm.x86.avx2.vpdpbsud.128
6390 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6391 // < 8 x i32> @llvm.x86.avx2.vpdpbsud.256
6392 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6393 // <16 x i32> @llvm.x86.avx10.vpdpbsud.512
6394 // (<16 x i32>, <64 x i8>, <64 x i8>)
6395 //
6396 // Multiply and Add Signed and Unsigned Bytes With Saturation
6397 // < 4 x i32> @llvm.x86.avx2.vpdpbsuds.128
6398 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6399 // < 8 x i32> @llvm.x86.avx2.vpdpbsuds.256
6400 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6401 // <16 x i32> @llvm.x86.avx512.vpdpbusds.512
6402 // (<16 x i32>, <64 x i8>, <64 x i8>)
6403 //
6404 // Multiply and Add Unsigned and Signed Bytes
6405 // < 4 x i32> @llvm.x86.avx512.vpdpbusd.128
6406 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6407 // < 8 x i32> @llvm.x86.avx512.vpdpbusd.256
6408 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6409 // <16 x i32> @llvm.x86.avx512.vpdpbusd.512
6410 // (<16 x i32>, <64 x i8>, <64 x i8>)
6411 //
6412 // Multiply and Add Unsigned and Signed Bytes With Saturation
6413 // < 4 x i32> @llvm.x86.avx512.vpdpbusds.128
6414 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6415 // < 8 x i32> @llvm.x86.avx512.vpdpbusds.256
6416 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6417 // <16 x i32> @llvm.x86.avx10.vpdpbsuds.512
6418 // (<16 x i32>, <64 x i8>, <64 x i8>)
6419 //
6420 // Multiply and Add Unsigned Bytes
6421 // < 4 x i32> @llvm.x86.avx2.vpdpbuud.128
6422 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6423 // < 8 x i32> @llvm.x86.avx2.vpdpbuud.256
6424 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6425 // <16 x i32> @llvm.x86.avx10.vpdpbuud.512
6426 // (<16 x i32>, <64 x i8>, <64 x i8>)
6427 //
6428 // Multiply and Add Unsigned Bytes With Saturation
6429 // < 4 x i32> @llvm.x86.avx2.vpdpbuuds.128
6430 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6431 // < 8 x i32> @llvm.x86.avx2.vpdpbuuds.256
6432 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6433 // <16 x i32> @llvm.x86.avx10.vpdpbuuds.512
6434 // (<16 x i32>, <64 x i8>, <64 x i8>)
6435 //
6436 // These intrinsics are auto-upgraded into non-masked forms:
6437 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128
6438 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6439 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128
6440 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6441 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256
6442 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6443 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256
6444 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6445 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512
6446 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6447 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512
6448 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6449 //
6450 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128
6451 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6452 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128
6453 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6454 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256
6455 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6456 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256
6457 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6458 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512
6459 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6460 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512
6461 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6462 case Intrinsic::x86_avx512_vpdpbusd_128:
6463 case Intrinsic::x86_avx512_vpdpbusd_256:
6464 case Intrinsic::x86_avx512_vpdpbusd_512:
6465 case Intrinsic::x86_avx512_vpdpbusds_128:
6466 case Intrinsic::x86_avx512_vpdpbusds_256:
6467 case Intrinsic::x86_avx512_vpdpbusds_512:
6468 case Intrinsic::x86_avx2_vpdpbssd_128:
6469 case Intrinsic::x86_avx2_vpdpbssd_256:
6470 case Intrinsic::x86_avx10_vpdpbssd_512:
6471 case Intrinsic::x86_avx2_vpdpbssds_128:
6472 case Intrinsic::x86_avx2_vpdpbssds_256:
6473 case Intrinsic::x86_avx10_vpdpbssds_512:
6474 case Intrinsic::x86_avx2_vpdpbsud_128:
6475 case Intrinsic::x86_avx2_vpdpbsud_256:
6476 case Intrinsic::x86_avx10_vpdpbsud_512:
6477 case Intrinsic::x86_avx2_vpdpbsuds_128:
6478 case Intrinsic::x86_avx2_vpdpbsuds_256:
6479 case Intrinsic::x86_avx10_vpdpbsuds_512:
6480 case Intrinsic::x86_avx2_vpdpbuud_128:
6481 case Intrinsic::x86_avx2_vpdpbuud_256:
6482 case Intrinsic::x86_avx10_vpdpbuud_512:
6483 case Intrinsic::x86_avx2_vpdpbuuds_128:
6484 case Intrinsic::x86_avx2_vpdpbuuds_256:
6485 case Intrinsic::x86_avx10_vpdpbuuds_512:
6486 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
6487 /*ZeroPurifies=*/true,
6488 /*EltSizeInBits=*/0,
6489 /*Lanes=*/kBothLanes);
6490 break;
6491
6492 // AVX Vector Neural Network Instructions: words
6493 //
6494 // Multiply and Add Signed Word Integers
6495 // < 4 x i32> @llvm.x86.avx512.vpdpwssd.128
6496 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6497 // < 8 x i32> @llvm.x86.avx512.vpdpwssd.256
6498 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6499 // <16 x i32> @llvm.x86.avx512.vpdpwssd.512
6500 // (<16 x i32>, <32 x i16>, <32 x i16>)
6501 //
6502 // Multiply and Add Signed Word Integers With Saturation
6503 // < 4 x i32> @llvm.x86.avx512.vpdpwssds.128
6504 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6505 // < 8 x i32> @llvm.x86.avx512.vpdpwssds.256
6506 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6507 // <16 x i32> @llvm.x86.avx512.vpdpwssds.512
6508 // (<16 x i32>, <32 x i16>, <32 x i16>)
6509 //
6510 // Multiply and Add Signed and Unsigned Word Integers
6511 // < 4 x i32> @llvm.x86.avx2.vpdpwsud.128
6512 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6513 // < 8 x i32> @llvm.x86.avx2.vpdpwsud.256
6514 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6515 // <16 x i32> @llvm.x86.avx10.vpdpwsud.512
6516 // (<16 x i32>, <32 x i16>, <32 x i16>)
6517 //
6518 // Multiply and Add Signed and Unsigned Word Integers With Saturation
6519 // < 4 x i32> @llvm.x86.avx2.vpdpwsuds.128
6520 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6521 // < 8 x i32> @llvm.x86.avx2.vpdpwsuds.256
6522 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6523 // <16 x i32> @llvm.x86.avx10.vpdpwsuds.512
6524 // (<16 x i32>, <32 x i16>, <32 x i16>)
6525 //
6526 // Multiply and Add Unsigned and Signed Word Integers
6527 // < 4 x i32> @llvm.x86.avx2.vpdpwusd.128
6528 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6529 // < 8 x i32> @llvm.x86.avx2.vpdpwusd.256
6530 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6531 // <16 x i32> @llvm.x86.avx10.vpdpwusd.512
6532 // (<16 x i32>, <32 x i16>, <32 x i16>)
6533 //
6534 // Multiply and Add Unsigned and Signed Word Integers With Saturation
6535 // < 4 x i32> @llvm.x86.avx2.vpdpwusds.128
6536 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6537 // < 8 x i32> @llvm.x86.avx2.vpdpwusds.256
6538 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6539 // <16 x i32> @llvm.x86.avx10.vpdpwusds.512
6540 // (<16 x i32>, <32 x i16>, <32 x i16>)
6541 //
6542 // Multiply and Add Unsigned and Unsigned Word Integers
6543 // < 4 x i32> @llvm.x86.avx2.vpdpwuud.128
6544 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6545 // < 8 x i32> @llvm.x86.avx2.vpdpwuud.256
6546 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6547 // <16 x i32> @llvm.x86.avx10.vpdpwuud.512
6548 // (<16 x i32>, <32 x i16>, <32 x i16>)
6549 //
6550 // Multiply and Add Unsigned and Unsigned Word Integers With Saturation
6551 // < 4 x i32> @llvm.x86.avx2.vpdpwuuds.128
6552 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6553 // < 8 x i32> @llvm.x86.avx2.vpdpwuuds.256
6554 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6555 // <16 x i32> @llvm.x86.avx10.vpdpwuuds.512
6556 // (<16 x i32>, <32 x i16>, <32 x i16>)
6557 //
6558 // These intrinsics are auto-upgraded into non-masked forms:
6559 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128
6560 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6561 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128
6562 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6563 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256
6564 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6565 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256
6566 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6567 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512
6568 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6569 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512
6570 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6571 //
6572 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128
6573 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6574 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128
6575 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6576 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256
6577 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6578 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256
6579 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6580 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512
6581 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6582 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512
6583 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6584 case Intrinsic::x86_avx512_vpdpwssd_128:
6585 case Intrinsic::x86_avx512_vpdpwssd_256:
6586 case Intrinsic::x86_avx512_vpdpwssd_512:
6587 case Intrinsic::x86_avx512_vpdpwssds_128:
6588 case Intrinsic::x86_avx512_vpdpwssds_256:
6589 case Intrinsic::x86_avx512_vpdpwssds_512:
6590 case Intrinsic::x86_avx2_vpdpwsud_128:
6591 case Intrinsic::x86_avx2_vpdpwsud_256:
6592 case Intrinsic::x86_avx10_vpdpwsud_512:
6593 case Intrinsic::x86_avx2_vpdpwsuds_128:
6594 case Intrinsic::x86_avx2_vpdpwsuds_256:
6595 case Intrinsic::x86_avx10_vpdpwsuds_512:
6596 case Intrinsic::x86_avx2_vpdpwusd_128:
6597 case Intrinsic::x86_avx2_vpdpwusd_256:
6598 case Intrinsic::x86_avx10_vpdpwusd_512:
6599 case Intrinsic::x86_avx2_vpdpwusds_128:
6600 case Intrinsic::x86_avx2_vpdpwusds_256:
6601 case Intrinsic::x86_avx10_vpdpwusds_512:
6602 case Intrinsic::x86_avx2_vpdpwuud_128:
6603 case Intrinsic::x86_avx2_vpdpwuud_256:
6604 case Intrinsic::x86_avx10_vpdpwuud_512:
6605 case Intrinsic::x86_avx2_vpdpwuuds_128:
6606 case Intrinsic::x86_avx2_vpdpwuuds_256:
6607 case Intrinsic::x86_avx10_vpdpwuuds_512:
6608 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6609 /*ZeroPurifies=*/true,
6610 /*EltSizeInBits=*/0,
6611 /*Lanes=*/kBothLanes);
6612 break;
6613
6614 // Dot Product of BF16 Pairs Accumulated Into Packed Single
6615 // Precision
6616 // <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128
6617 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6618 // <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256
6619 // (<8 x float>, <16 x bfloat>, <16 x bfloat>)
6620 // <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512
6621 // (<16 x float>, <32 x bfloat>, <32 x bfloat>)
6622 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6623 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6624 case Intrinsic::x86_avx512bf16_dpbf16ps_512:
6625 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6626 /*ZeroPurifies=*/false,
6627 /*EltSizeInBits=*/0,
6628 /*Lanes=*/kBothLanes);
6629 break;
6630
6631 case Intrinsic::x86_sse_cmp_ss:
6632 case Intrinsic::x86_sse2_cmp_sd:
6633 case Intrinsic::x86_sse_comieq_ss:
6634 case Intrinsic::x86_sse_comilt_ss:
6635 case Intrinsic::x86_sse_comile_ss:
6636 case Intrinsic::x86_sse_comigt_ss:
6637 case Intrinsic::x86_sse_comige_ss:
6638 case Intrinsic::x86_sse_comineq_ss:
6639 case Intrinsic::x86_sse_ucomieq_ss:
6640 case Intrinsic::x86_sse_ucomilt_ss:
6641 case Intrinsic::x86_sse_ucomile_ss:
6642 case Intrinsic::x86_sse_ucomigt_ss:
6643 case Intrinsic::x86_sse_ucomige_ss:
6644 case Intrinsic::x86_sse_ucomineq_ss:
6645 case Intrinsic::x86_sse2_comieq_sd:
6646 case Intrinsic::x86_sse2_comilt_sd:
6647 case Intrinsic::x86_sse2_comile_sd:
6648 case Intrinsic::x86_sse2_comigt_sd:
6649 case Intrinsic::x86_sse2_comige_sd:
6650 case Intrinsic::x86_sse2_comineq_sd:
6651 case Intrinsic::x86_sse2_ucomieq_sd:
6652 case Intrinsic::x86_sse2_ucomilt_sd:
6653 case Intrinsic::x86_sse2_ucomile_sd:
6654 case Intrinsic::x86_sse2_ucomigt_sd:
6655 case Intrinsic::x86_sse2_ucomige_sd:
6656 case Intrinsic::x86_sse2_ucomineq_sd:
6657 handleVectorCompareScalarIntrinsic(I);
6658 break;
6659
6660 case Intrinsic::x86_avx_cmp_pd_256:
6661 case Intrinsic::x86_avx_cmp_ps_256:
6662 case Intrinsic::x86_sse2_cmp_pd:
6663 case Intrinsic::x86_sse_cmp_ps:
6664 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/true);
6665 break;
6666
6667 case Intrinsic::x86_bmi_bextr_32:
6668 case Intrinsic::x86_bmi_bextr_64:
6669 case Intrinsic::x86_bmi_bzhi_32:
6670 case Intrinsic::x86_bmi_bzhi_64:
6671 handleGenericBitManipulation(I);
6672 break;
6673
6674 case Intrinsic::x86_pclmulqdq:
6675 case Intrinsic::x86_pclmulqdq_256:
6676 case Intrinsic::x86_pclmulqdq_512:
6677 handlePclmulIntrinsic(I);
6678 break;
6679
6680 case Intrinsic::x86_avx_round_pd_256:
6681 case Intrinsic::x86_avx_round_ps_256:
6682 case Intrinsic::x86_sse41_round_pd:
6683 case Intrinsic::x86_sse41_round_ps:
6684 handleRoundPdPsIntrinsic(I);
6685 break;
6686
6687 case Intrinsic::x86_sse41_round_sd:
6688 case Intrinsic::x86_sse41_round_ss:
6689 handleUnarySdSsIntrinsic(I);
6690 break;
6691
6692 case Intrinsic::x86_sse2_max_sd:
6693 case Intrinsic::x86_sse_max_ss:
6694 case Intrinsic::x86_sse2_min_sd:
6695 case Intrinsic::x86_sse_min_ss:
6696 handleBinarySdSsIntrinsic(I);
6697 break;
6698
6699 case Intrinsic::x86_avx_vtestc_pd:
6700 case Intrinsic::x86_avx_vtestc_pd_256:
6701 case Intrinsic::x86_avx_vtestc_ps:
6702 case Intrinsic::x86_avx_vtestc_ps_256:
6703 case Intrinsic::x86_avx_vtestnzc_pd:
6704 case Intrinsic::x86_avx_vtestnzc_pd_256:
6705 case Intrinsic::x86_avx_vtestnzc_ps:
6706 case Intrinsic::x86_avx_vtestnzc_ps_256:
6707 case Intrinsic::x86_avx_vtestz_pd:
6708 case Intrinsic::x86_avx_vtestz_pd_256:
6709 case Intrinsic::x86_avx_vtestz_ps:
6710 case Intrinsic::x86_avx_vtestz_ps_256:
6711 case Intrinsic::x86_avx_ptestc_256:
6712 case Intrinsic::x86_avx_ptestnzc_256:
6713 case Intrinsic::x86_avx_ptestz_256:
6714 case Intrinsic::x86_sse41_ptestc:
6715 case Intrinsic::x86_sse41_ptestnzc:
6716 case Intrinsic::x86_sse41_ptestz:
6717 handleVtestIntrinsic(I);
6718 break;
6719
6720 // Packed Horizontal Add/Subtract
6721 case Intrinsic::x86_ssse3_phadd_w:
6722 case Intrinsic::x86_ssse3_phadd_w_128:
6723 case Intrinsic::x86_ssse3_phsub_w:
6724 case Intrinsic::x86_ssse3_phsub_w_128:
6725 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6726 /*ReinterpretElemWidth=*/16);
6727 break;
6728
6729 case Intrinsic::x86_avx2_phadd_w:
6730 case Intrinsic::x86_avx2_phsub_w:
6731 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6732 /*ReinterpretElemWidth=*/16);
6733 break;
6734
6735 // Packed Horizontal Add/Subtract
6736 case Intrinsic::x86_ssse3_phadd_d:
6737 case Intrinsic::x86_ssse3_phadd_d_128:
6738 case Intrinsic::x86_ssse3_phsub_d:
6739 case Intrinsic::x86_ssse3_phsub_d_128:
6740 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6741 /*ReinterpretElemWidth=*/32);
6742 break;
6743
6744 case Intrinsic::x86_avx2_phadd_d:
6745 case Intrinsic::x86_avx2_phsub_d:
6746 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6747 /*ReinterpretElemWidth=*/32);
6748 break;
6749
6750 // Packed Horizontal Add/Subtract and Saturate
6751 case Intrinsic::x86_ssse3_phadd_sw:
6752 case Intrinsic::x86_ssse3_phadd_sw_128:
6753 case Intrinsic::x86_ssse3_phsub_sw:
6754 case Intrinsic::x86_ssse3_phsub_sw_128:
6755 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6756 /*ReinterpretElemWidth=*/16);
6757 break;
6758
6759 case Intrinsic::x86_avx2_phadd_sw:
6760 case Intrinsic::x86_avx2_phsub_sw:
6761 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6762 /*ReinterpretElemWidth=*/16);
6763 break;
6764
6765 // Packed Single/Double Precision Floating-Point Horizontal Add
6766 case Intrinsic::x86_sse3_hadd_ps:
6767 case Intrinsic::x86_sse3_hadd_pd:
6768 case Intrinsic::x86_sse3_hsub_ps:
6769 case Intrinsic::x86_sse3_hsub_pd:
6770 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
6771 break;
6772
6773 case Intrinsic::x86_avx_hadd_pd_256:
6774 case Intrinsic::x86_avx_hadd_ps_256:
6775 case Intrinsic::x86_avx_hsub_pd_256:
6776 case Intrinsic::x86_avx_hsub_ps_256:
6777 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2);
6778 break;
6779
6780 case Intrinsic::x86_avx_maskstore_ps:
6781 case Intrinsic::x86_avx_maskstore_pd:
6782 case Intrinsic::x86_avx_maskstore_ps_256:
6783 case Intrinsic::x86_avx_maskstore_pd_256:
6784 case Intrinsic::x86_avx2_maskstore_d:
6785 case Intrinsic::x86_avx2_maskstore_q:
6786 case Intrinsic::x86_avx2_maskstore_d_256:
6787 case Intrinsic::x86_avx2_maskstore_q_256: {
6788 handleAVXMaskedStore(I);
6789 break;
6790 }
6791
6792 case Intrinsic::x86_avx_maskload_ps:
6793 case Intrinsic::x86_avx_maskload_pd:
6794 case Intrinsic::x86_avx_maskload_ps_256:
6795 case Intrinsic::x86_avx_maskload_pd_256:
6796 case Intrinsic::x86_avx2_maskload_d:
6797 case Intrinsic::x86_avx2_maskload_q:
6798 case Intrinsic::x86_avx2_maskload_d_256:
6799 case Intrinsic::x86_avx2_maskload_q_256: {
6800 handleAVXMaskedLoad(I);
6801 break;
6802 }
6803
6804 // Packed
6805 case Intrinsic::x86_avx512fp16_add_ph_512:
6806 case Intrinsic::x86_avx512fp16_sub_ph_512:
6807 case Intrinsic::x86_avx512fp16_mul_ph_512:
6808 case Intrinsic::x86_avx512fp16_div_ph_512:
6809 case Intrinsic::x86_avx512fp16_max_ph_512:
6810 case Intrinsic::x86_avx512fp16_min_ph_512:
6811 case Intrinsic::x86_avx512_min_ps_512:
6812 case Intrinsic::x86_avx512_min_pd_512:
6813 case Intrinsic::x86_avx512_max_ps_512:
6814 case Intrinsic::x86_avx512_max_pd_512: {
6815 // These AVX512 variants contain the rounding mode as a trailing flag.
6816 // Earlier variants do not have a trailing flag and are already handled
6817 // by maybeHandleSimpleNomemIntrinsic(I, 0) via
6818 // maybeHandleUnknownIntrinsic.
6819 [[maybe_unused]] bool Success =
6820 maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/1);
6821 assert(Success);
6822 break;
6823 }
6824
6825 case Intrinsic::x86_avx_vpermilvar_pd:
6826 case Intrinsic::x86_avx_vpermilvar_pd_256:
6827 case Intrinsic::x86_avx512_vpermilvar_pd_512:
6828 case Intrinsic::x86_avx_vpermilvar_ps:
6829 case Intrinsic::x86_avx_vpermilvar_ps_256:
6830 case Intrinsic::x86_avx512_vpermilvar_ps_512: {
6831 handleAVXVpermilvar(I);
6832 break;
6833 }
6834
6835 case Intrinsic::x86_avx512_vpermi2var_d_128:
6836 case Intrinsic::x86_avx512_vpermi2var_d_256:
6837 case Intrinsic::x86_avx512_vpermi2var_d_512:
6838 case Intrinsic::x86_avx512_vpermi2var_hi_128:
6839 case Intrinsic::x86_avx512_vpermi2var_hi_256:
6840 case Intrinsic::x86_avx512_vpermi2var_hi_512:
6841 case Intrinsic::x86_avx512_vpermi2var_pd_128:
6842 case Intrinsic::x86_avx512_vpermi2var_pd_256:
6843 case Intrinsic::x86_avx512_vpermi2var_pd_512:
6844 case Intrinsic::x86_avx512_vpermi2var_ps_128:
6845 case Intrinsic::x86_avx512_vpermi2var_ps_256:
6846 case Intrinsic::x86_avx512_vpermi2var_ps_512:
6847 case Intrinsic::x86_avx512_vpermi2var_q_128:
6848 case Intrinsic::x86_avx512_vpermi2var_q_256:
6849 case Intrinsic::x86_avx512_vpermi2var_q_512:
6850 case Intrinsic::x86_avx512_vpermi2var_qi_128:
6851 case Intrinsic::x86_avx512_vpermi2var_qi_256:
6852 case Intrinsic::x86_avx512_vpermi2var_qi_512:
6853 handleAVXVpermi2var(I);
6854 break;
6855
6856 // Packed Shuffle
6857 // llvm.x86.sse.pshuf.w(<1 x i64>, i8)
6858 // llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>)
6859 // llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
6860 // llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
6861 // llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
6862 //
6863 // The following intrinsics are auto-upgraded:
6864 // llvm.x86.sse2.pshuf.d(<4 x i32>, i8)
6865 // llvm.x86.sse2.gpshufh.w(<8 x i16>, i8)
6866 // llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
6867 case Intrinsic::x86_avx2_pshuf_b:
6868 case Intrinsic::x86_sse_pshuf_w:
6869 case Intrinsic::x86_ssse3_pshuf_b_128:
6870 case Intrinsic::x86_ssse3_pshuf_b:
6871 case Intrinsic::x86_avx512_pshuf_b_512:
6872 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6873 /*trailingVerbatimArgs=*/1,
6874 /*forceIntegerIntrinsic=*/false);
6875 break;
6876
6877 // AVX512 PMOV: Packed MOV, with truncation
6878 // Precisely handled by applying the same intrinsic to the shadow
6879 case Intrinsic::x86_avx512_mask_pmov_dw_128:
6880 case Intrinsic::x86_avx512_mask_pmov_db_128:
6881 case Intrinsic::x86_avx512_mask_pmov_qb_128:
6882 case Intrinsic::x86_avx512_mask_pmov_qw_128:
6883 case Intrinsic::x86_avx512_mask_pmov_qd_128:
6884 case Intrinsic::x86_avx512_mask_pmov_wb_128:
6885 case Intrinsic::x86_avx512_mask_pmov_dw_256:
6886 case Intrinsic::x86_avx512_mask_pmov_db_256:
6887 case Intrinsic::x86_avx512_mask_pmov_qb_256:
6888 case Intrinsic::x86_avx512_mask_pmov_qw_256:
6889 case Intrinsic::x86_avx512_mask_pmov_dw_512:
6890 case Intrinsic::x86_avx512_mask_pmov_db_512:
6891 case Intrinsic::x86_avx512_mask_pmov_qb_512:
6892 case Intrinsic::x86_avx512_mask_pmov_qw_512: {
6893 // Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} were removed in
6894 // f608dc1f5775ee880e8ea30e2d06ab5a4a935c22
6895 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6896 /*trailingVerbatimArgs=*/1,
6897 /*forceIntegerIntrinsic=*/false);
6898 break;
6899 }
6900
6901 // AVX512 PMOV{S,US}: Packed MOV, with signed/unsigned saturation
6902 // Approximately handled using the corresponding truncation intrinsic
6903 // TODO: improve handleAVX512VectorDownConvert to precisely model saturation
6904 case Intrinsic::x86_avx512_mask_pmovs_dw_512:
6905 case Intrinsic::x86_avx512_mask_pmovus_dw_512: {
6906 handleIntrinsicByApplyingToShadow(
6907 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_512,
6908 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6909 break;
6910 }
6911
6912 case Intrinsic::x86_avx512_mask_pmovs_dw_256:
6913 case Intrinsic::x86_avx512_mask_pmovus_dw_256:
6914 handleIntrinsicByApplyingToShadow(
6915 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_256,
6916 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6917 break;
6918
6919 case Intrinsic::x86_avx512_mask_pmovs_dw_128:
6920 case Intrinsic::x86_avx512_mask_pmovus_dw_128:
6921 handleIntrinsicByApplyingToShadow(
6922 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_128,
6923 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6924 break;
6925
6926 case Intrinsic::x86_avx512_mask_pmovs_db_512:
6927 case Intrinsic::x86_avx512_mask_pmovus_db_512: {
6928 handleIntrinsicByApplyingToShadow(
6929 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_512,
6930 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6931 break;
6932 }
6933
6934 case Intrinsic::x86_avx512_mask_pmovs_db_256:
6935 case Intrinsic::x86_avx512_mask_pmovus_db_256:
6936 handleIntrinsicByApplyingToShadow(
6937 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_256,
6938 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6939 break;
6940
6941 case Intrinsic::x86_avx512_mask_pmovs_db_128:
6942 case Intrinsic::x86_avx512_mask_pmovus_db_128:
6943 handleIntrinsicByApplyingToShadow(
6944 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_128,
6945 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6946 break;
6947
6948 case Intrinsic::x86_avx512_mask_pmovs_qb_512:
6949 case Intrinsic::x86_avx512_mask_pmovus_qb_512: {
6950 handleIntrinsicByApplyingToShadow(
6951 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_512,
6952 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6953 break;
6954 }
6955
6956 case Intrinsic::x86_avx512_mask_pmovs_qb_256:
6957 case Intrinsic::x86_avx512_mask_pmovus_qb_256:
6958 handleIntrinsicByApplyingToShadow(
6959 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_256,
6960 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6961 break;
6962
6963 case Intrinsic::x86_avx512_mask_pmovs_qb_128:
6964 case Intrinsic::x86_avx512_mask_pmovus_qb_128:
6965 handleIntrinsicByApplyingToShadow(
6966 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_128,
6967 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6968 break;
6969
6970 case Intrinsic::x86_avx512_mask_pmovs_qw_512:
6971 case Intrinsic::x86_avx512_mask_pmovus_qw_512: {
6972 handleIntrinsicByApplyingToShadow(
6973 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_512,
6974 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6975 break;
6976 }
6977
6978 case Intrinsic::x86_avx512_mask_pmovs_qw_256:
6979 case Intrinsic::x86_avx512_mask_pmovus_qw_256:
6980 handleIntrinsicByApplyingToShadow(
6981 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_256,
6982 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6983 break;
6984
6985 case Intrinsic::x86_avx512_mask_pmovs_qw_128:
6986 case Intrinsic::x86_avx512_mask_pmovus_qw_128:
6987 handleIntrinsicByApplyingToShadow(
6988 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_128,
6989 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6990 break;
6991
6992 case Intrinsic::x86_avx512_mask_pmovs_qd_128:
6993 case Intrinsic::x86_avx512_mask_pmovus_qd_128:
6994 handleIntrinsicByApplyingToShadow(
6995 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qd_128,
6996 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
6997 break;
6998
6999 case Intrinsic::x86_avx512_mask_pmovs_wb_128:
7000 case Intrinsic::x86_avx512_mask_pmovus_wb_128:
7001 handleIntrinsicByApplyingToShadow(
7002 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_wb_128,
7003 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
7004 break;
7005
7006 case Intrinsic::x86_avx512_mask_pmovs_qd_256:
7007 case Intrinsic::x86_avx512_mask_pmovus_qd_256:
7008 case Intrinsic::x86_avx512_mask_pmovs_wb_256:
7009 case Intrinsic::x86_avx512_mask_pmovus_wb_256:
7010 case Intrinsic::x86_avx512_mask_pmovs_qd_512:
7011 case Intrinsic::x86_avx512_mask_pmovus_qd_512:
7012 case Intrinsic::x86_avx512_mask_pmovs_wb_512:
7013 case Intrinsic::x86_avx512_mask_pmovus_wb_512: {
7014 // Since Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} do not exist,
7015 // we cannot use handleIntrinsicByApplyingToShadow. Instead, we call the
7016 // slow-path handler.
7017 handleAVX512VectorDownConvert(I);
7018 break;
7019 }
7020
7021 // e.g.,
7022 // <16 x float> @llvm.x86.avx512.mask.compress
7023 // (<16 x float> %data, <16 x float> %passthru,
7024 // <16 x i1> %mask)
7025 // <16 x i32> @llvm.x86.avx512.mask.compress
7026 // (<16 x i32> %data, <16 x i32> %passthru,
7027 // <16 x i1> %mask)
7028 case Intrinsic::x86_avx512_mask_compress:
7029 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7030 /*trailingVerbatimArgs=*/1,
7031 /*forceIntegerIntrinsic=*/true);
7032 break;
7033
7034 // AVX512/AVX10 Reciprocal
7035 // <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
7036 // (<16 x float>, <16 x float>, i16)
7037 // <8 x float> @llvm.x86.avx512.rsqrt14.ps.256
7038 // (<8 x float>, <8 x float>, i8)
7039 // <4 x float> @llvm.x86.avx512.rsqrt14.ps.128
7040 // (<4 x float>, <4 x float>, i8)
7041 //
7042 // <8 x double> @llvm.x86.avx512.rsqrt14.pd.512
7043 // (<8 x double>, <8 x double>, i8)
7044 // <4 x double> @llvm.x86.avx512.rsqrt14.pd.256
7045 // (<4 x double>, <4 x double>, i8)
7046 // <2 x double> @llvm.x86.avx512.rsqrt14.pd.128
7047 // (<2 x double>, <2 x double>, i8)
7048 //
7049 // <32 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.512
7050 // (<32 x bfloat>, <32 x bfloat>, i32)
7051 // <16 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.256
7052 // (<16 x bfloat>, <16 x bfloat>, i16)
7053 // <8 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.128
7054 // (<8 x bfloat>, <8 x bfloat>, i8)
7055 //
7056 // <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512
7057 // (<32 x half>, <32 x half>, i32)
7058 // <16 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.256
7059 // (<16 x half>, <16 x half>, i16)
7060 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.128
7061 // (<8 x half>, <8 x half>, i8)
7062 //
7063 // TODO: 3-operand variants are not handled:
7064 // <2 x double> @llvm.x86.avx512.rsqrt14.sd
7065 // (<2 x double>, <2 x double>, <2 x double>, i8)
7066 // <4 x float> @llvm.x86.avx512.rsqrt14.ss
7067 // (<4 x float>, <4 x float>, <4 x float>, i8)
7068 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh
7069 // (<8 x half>, <8 x half>, <8 x half>, i8)
7070 case Intrinsic::x86_avx512_rsqrt14_ps_512:
7071 case Intrinsic::x86_avx512_rsqrt14_ps_256:
7072 case Intrinsic::x86_avx512_rsqrt14_ps_128:
7073 case Intrinsic::x86_avx512_rsqrt14_pd_512:
7074 case Intrinsic::x86_avx512_rsqrt14_pd_256:
7075 case Intrinsic::x86_avx512_rsqrt14_pd_128:
7076 case Intrinsic::x86_avx10_mask_rsqrt_bf16_512:
7077 case Intrinsic::x86_avx10_mask_rsqrt_bf16_256:
7078 case Intrinsic::x86_avx10_mask_rsqrt_bf16_128:
7079 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_512:
7080 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_256:
7081 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_128:
7082 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7083 /*WriteThruIndex=*/1,
7084 /*MaskIndex=*/2);
7085 break;
7086
7087 // AVX512/AVX10 Reciprocal Square Root
7088 // <16 x float> @llvm.x86.avx512.rcp14.ps.512
7089 // (<16 x float>, <16 x float>, i16)
7090 // <8 x float> @llvm.x86.avx512.rcp14.ps.256
7091 // (<8 x float>, <8 x float>, i8)
7092 // <4 x float> @llvm.x86.avx512.rcp14.ps.128
7093 // (<4 x float>, <4 x float>, i8)
7094 //
7095 // <8 x double> @llvm.x86.avx512.rcp14.pd.512
7096 // (<8 x double>, <8 x double>, i8)
7097 // <4 x double> @llvm.x86.avx512.rcp14.pd.256
7098 // (<4 x double>, <4 x double>, i8)
7099 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
7100 // (<2 x double>, <2 x double>, i8)
7101 //
7102 // <32 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.512
7103 // (<32 x bfloat>, <32 x bfloat>, i32)
7104 // <16 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.256
7105 // (<16 x bfloat>, <16 x bfloat>, i16)
7106 // <8 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.128
7107 // (<8 x bfloat>, <8 x bfloat>, i8)
7108 //
7109 // <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512
7110 // (<32 x half>, <32 x half>, i32)
7111 // <16 x half> @llvm.x86.avx512fp16.mask.rcp.ph.256
7112 // (<16 x half>, <16 x half>, i16)
7113 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.ph.128
7114 // (<8 x half>, <8 x half>, i8)
7115 //
7116 // TODO: 3-operand variants are not handled:
7117 // <2 x double> @llvm.x86.avx512.rcp14.sd
7118 // (<2 x double>, <2 x double>, <2 x double>, i8)
7119 // <4 x float> @llvm.x86.avx512.rcp14.ss
7120 // (<4 x float>, <4 x float>, <4 x float>, i8)
7121 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh
7122 // (<8 x half>, <8 x half>, <8 x half>, i8)
7123 case Intrinsic::x86_avx512_rcp14_ps_512:
7124 case Intrinsic::x86_avx512_rcp14_ps_256:
7125 case Intrinsic::x86_avx512_rcp14_ps_128:
7126 case Intrinsic::x86_avx512_rcp14_pd_512:
7127 case Intrinsic::x86_avx512_rcp14_pd_256:
7128 case Intrinsic::x86_avx512_rcp14_pd_128:
7129 case Intrinsic::x86_avx10_mask_rcp_bf16_512:
7130 case Intrinsic::x86_avx10_mask_rcp_bf16_256:
7131 case Intrinsic::x86_avx10_mask_rcp_bf16_128:
7132 case Intrinsic::x86_avx512fp16_mask_rcp_ph_512:
7133 case Intrinsic::x86_avx512fp16_mask_rcp_ph_256:
7134 case Intrinsic::x86_avx512fp16_mask_rcp_ph_128:
7135 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7136 /*WriteThruIndex=*/1,
7137 /*MaskIndex=*/2);
7138 break;
7139
7140 // <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512
7141 // (<32 x half>, i32, <32 x half>, i32, i32)
7142 // <16 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.256
7143 // (<16 x half>, i32, <16 x half>, i32, i16)
7144 // <8 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.128
7145 // (<8 x half>, i32, <8 x half>, i32, i8)
7146 //
7147 // <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512
7148 // (<16 x float>, i32, <16 x float>, i16, i32)
7149 // <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256
7150 // (<8 x float>, i32, <8 x float>, i8)
7151 // <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128
7152 // (<4 x float>, i32, <4 x float>, i8)
7153 //
7154 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
7155 // (<8 x double>, i32, <8 x double>, i8, i32)
7156 // A Imm WriteThru Mask Rounding
7157 // <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256
7158 // (<4 x double>, i32, <4 x double>, i8)
7159 // <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128
7160 // (<2 x double>, i32, <2 x double>, i8)
7161 // A Imm WriteThru Mask
7162 //
7163 // <32 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.512
7164 // (<32 x bfloat>, i32, <32 x bfloat>, i32)
7165 // <16 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.256
7166 // (<16 x bfloat>, i32, <16 x bfloat>, i16)
7167 // <8 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.128
7168 // (<8 x bfloat>, i32, <8 x bfloat>, i8)
7169 //
7170 // Not supported: three vectors
7171 // - <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh
7172 // (<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
7173 // - <4 x float> @llvm.x86.avx512.mask.rndscale.ss
7174 // (<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
7175 // - <2 x double> @llvm.x86.avx512.mask.rndscale.sd
7176 // (<2 x double>, <2 x double>, <2 x double>, i8, i32,
7177 // i32)
7178 // A B WriteThru Mask Imm
7179 // Rounding
7180 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_512:
7181 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_256:
7182 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_128:
7183 case Intrinsic::x86_avx512_mask_rndscale_ps_512:
7184 case Intrinsic::x86_avx512_mask_rndscale_ps_256:
7185 case Intrinsic::x86_avx512_mask_rndscale_ps_128:
7186 case Intrinsic::x86_avx512_mask_rndscale_pd_512:
7187 case Intrinsic::x86_avx512_mask_rndscale_pd_256:
7188 case Intrinsic::x86_avx512_mask_rndscale_pd_128:
7189 case Intrinsic::x86_avx10_mask_rndscale_bf16_512:
7190 case Intrinsic::x86_avx10_mask_rndscale_bf16_256:
7191 case Intrinsic::x86_avx10_mask_rndscale_bf16_128:
7192 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7193 /*WriteThruIndex=*/2,
7194 /*MaskIndex=*/3);
7195 break;
7196
7197 // AVX512 Vector Scale Float* Packed
7198 //
7199 // < 8 x double> @llvm.x86.avx512.mask.scalef.pd.512
7200 // (<8 x double>, <8 x double>, <8 x double>, i8, i32)
7201 // A B WriteThru Msk Round
7202 // < 4 x double> @llvm.x86.avx512.mask.scalef.pd.256
7203 // (<4 x double>, <4 x double>, <4 x double>, i8)
7204 // < 2 x double> @llvm.x86.avx512.mask.scalef.pd.128
7205 // (<2 x double>, <2 x double>, <2 x double>, i8)
7206 //
7207 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
7208 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
7209 // < 8 x float> @llvm.x86.avx512.mask.scalef.ps.256
7210 // (<8 x float>, <8 x float>, <8 x float>, i8)
7211 // < 4 x float> @llvm.x86.avx512.mask.scalef.ps.128
7212 // (<4 x float>, <4 x float>, <4 x float>, i8)
7213 //
7214 // <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512
7215 // (<32 x half>, <32 x half>, <32 x half>, i32, i32)
7216 // <16 x half> @llvm.x86.avx512fp16.mask.scalef.ph.256
7217 // (<16 x half>, <16 x half>, <16 x half>, i16)
7218 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.ph.128
7219 // (<8 x half>, <8 x half>, <8 x half>, i8)
7220 //
7221 // TODO: AVX10
7222 // <32 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.512
7223 // (<32 x bfloat>, <32 x bfloat>, <32 x bfloat>, i32)
7224 // <16 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.256
7225 // (<16 x bfloat>, <16 x bfloat>, <16 x bfloat>, i16)
7226 // < 8 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.128
7227 // (<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i8)
7228 case Intrinsic::x86_avx512_mask_scalef_pd_512:
7229 case Intrinsic::x86_avx512_mask_scalef_pd_256:
7230 case Intrinsic::x86_avx512_mask_scalef_pd_128:
7231 case Intrinsic::x86_avx512_mask_scalef_ps_512:
7232 case Intrinsic::x86_avx512_mask_scalef_ps_256:
7233 case Intrinsic::x86_avx512_mask_scalef_ps_128:
7234 case Intrinsic::x86_avx512fp16_mask_scalef_ph_512:
7235 case Intrinsic::x86_avx512fp16_mask_scalef_ph_256:
7236 case Intrinsic::x86_avx512fp16_mask_scalef_ph_128:
7237 // The AVX512 512-bit operand variants have an extra operand (the
7238 // Rounding mode). The extra operand, if present, will be
7239 // automatically checked by the handler.
7240 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0, 1},
7241 /*WriteThruIndex=*/2,
7242 /*MaskIndex=*/3);
7243 break;
7244
7245 // TODO: AVX512 Vector Scale Float* Scalar
7246 //
7247 // This is different from the Packed variant, because some bits are copied,
7248 // and some bits are zeroed.
7249 //
7250 // < 4 x float> @llvm.x86.avx512.mask.scalef.ss
7251 // (<4 x float>, <4 x float>, <4 x float>, i8, i32)
7252 //
7253 // < 2 x double> @llvm.x86.avx512.mask.scalef.sd
7254 // (<2 x double>, <2 x double>, <2 x double>, i8, i32)
7255 //
7256 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.sh
7257 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
7258
7259 // AVX512 FP16 Arithmetic
7260 case Intrinsic::x86_avx512fp16_mask_add_sh_round:
7261 case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
7262 case Intrinsic::x86_avx512fp16_mask_mul_sh_round:
7263 case Intrinsic::x86_avx512fp16_mask_div_sh_round:
7264 case Intrinsic::x86_avx512fp16_mask_max_sh_round:
7265 case Intrinsic::x86_avx512fp16_mask_min_sh_round: {
7266 visitGenericScalarHalfwordInst(I);
7267 break;
7268 }
7269
7270 // AVX512 Floating-Point Classification
7271 // - <8 x i1> @llvm.x86.avx512.fpclass.pd.512(<8 x double>, i32)
7272 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512(<16 x float>, i32)
7273 case Intrinsic::x86_avx512_fpclass_pd_512:
7274 case Intrinsic::x86_avx512_fpclass_ps_512:
7275 handleAVX512FPClass(I);
7276 break;
7277
7278 // AVX Galois Field New Instructions
7279 case Intrinsic::x86_vgf2p8affineqb_128:
7280 case Intrinsic::x86_vgf2p8affineqb_256:
7281 case Intrinsic::x86_vgf2p8affineqb_512:
7282 handleAVXGF2P8Affine(I);
7283 break;
7284
7285 default:
7286 return false;
7287 }
7288
7289 return true;
7290 }
7291
7292 bool maybeHandleArmSIMDIntrinsic(IntrinsicInst &I) {
7293 switch (I.getIntrinsicID()) {
7294 // Two operands e.g.,
7295 // - <8 x i8> @llvm.aarch64.neon.rshrn.v8i8 (<8 x i16>, i32)
7296 // - <4 x i16> @llvm.aarch64.neon.uqrshl.v4i16(<4 x i16>, <4 x i16>)
7297 case Intrinsic::aarch64_neon_rshrn:
7298 case Intrinsic::aarch64_neon_sqrshl:
7299 case Intrinsic::aarch64_neon_sqrshrn:
7300 case Intrinsic::aarch64_neon_sqrshrun:
7301 case Intrinsic::aarch64_neon_sqshl:
7302 case Intrinsic::aarch64_neon_sqshlu:
7303 case Intrinsic::aarch64_neon_sqshrn:
7304 case Intrinsic::aarch64_neon_sqshrun:
7305 case Intrinsic::aarch64_neon_srshl:
7306 case Intrinsic::aarch64_neon_sshl:
7307 case Intrinsic::aarch64_neon_uqrshl:
7308 case Intrinsic::aarch64_neon_uqrshrn:
7309 case Intrinsic::aarch64_neon_uqshl:
7310 case Intrinsic::aarch64_neon_uqshrn:
7311 case Intrinsic::aarch64_neon_urshl:
7312 case Intrinsic::aarch64_neon_ushl:
7313 handleVectorShiftIntrinsic(I, /* Variable */ false);
7314 break;
7315
7316 // Vector Shift Left/Right and Insert
7317 //
7318 // Three operands e.g.,
7319 // - <4 x i16> @llvm.aarch64.neon.vsli.v4i16
7320 // (<4 x i16> %a, <4 x i16> %b, i32 %n)
7321 // - <16 x i8> @llvm.aarch64.neon.vsri.v16i8
7322 // (<16 x i8> %a, <16 x i8> %b, i32 %n)
7323 //
7324 // %b is shifted by %n bits, and the "missing" bits are filled in with %a
7325 // (instead of zero-extending/sign-extending).
7326 case Intrinsic::aarch64_neon_vsli:
7327 case Intrinsic::aarch64_neon_vsri:
7328 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7329 /*trailingVerbatimArgs=*/1,
7330 /*forceIntegerIntrinsic=*/false);
7331 break;
7332
7333 // TODO: handling max/min similarly to AND/OR may be more precise
7334 // Floating-Point Maximum/Minimum Pairwise
7335 case Intrinsic::aarch64_neon_fmaxp:
7336 case Intrinsic::aarch64_neon_fminp:
7337 // Floating-Point Maximum/Minimum Number Pairwise
7338 case Intrinsic::aarch64_neon_fmaxnmp:
7339 case Intrinsic::aarch64_neon_fminnmp:
7340 // Signed/Unsigned Maximum/Minimum Pairwise
7341 case Intrinsic::aarch64_neon_smaxp:
7342 case Intrinsic::aarch64_neon_sminp:
7343 case Intrinsic::aarch64_neon_umaxp:
7344 case Intrinsic::aarch64_neon_uminp:
7345 // Add Pairwise
7346 case Intrinsic::aarch64_neon_addp:
7347 // Floating-point Add Pairwise
7348 case Intrinsic::aarch64_neon_faddp:
7349 // Add Long Pairwise
7350 case Intrinsic::aarch64_neon_saddlp:
7351 case Intrinsic::aarch64_neon_uaddlp: {
7352 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
7353 break;
7354 }
7355
7356 // Floating-point Convert to integer, rounding to nearest with ties to Away
7357 case Intrinsic::aarch64_neon_fcvtas:
7358 case Intrinsic::aarch64_neon_fcvtau:
7359 // Floating-point convert to integer, rounding toward minus infinity
7360 case Intrinsic::aarch64_neon_fcvtms:
7361 case Intrinsic::aarch64_neon_fcvtmu:
7362 // Floating-point convert to integer, rounding to nearest with ties to even
7363 case Intrinsic::aarch64_neon_fcvtns:
7364 case Intrinsic::aarch64_neon_fcvtnu:
7365 // Floating-point convert to integer, rounding toward plus infinity
7366 case Intrinsic::aarch64_neon_fcvtps:
7367 case Intrinsic::aarch64_neon_fcvtpu:
7368 // Floating-point Convert to integer, rounding toward Zero
7369 case Intrinsic::aarch64_neon_fcvtzs:
7370 case Intrinsic::aarch64_neon_fcvtzu:
7371 // Floating-point convert to lower precision narrow, rounding to odd
7372 case Intrinsic::aarch64_neon_fcvtxn:
7373 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
7374 break;
7375
7376 // Vector Conversions Between Fixed-Point and Floating-Point
7377 case Intrinsic::aarch64_neon_vcvtfxs2fp:
7378 case Intrinsic::aarch64_neon_vcvtfp2fxs:
7379 case Intrinsic::aarch64_neon_vcvtfxu2fp:
7380 case Intrinsic::aarch64_neon_vcvtfp2fxu:
7381 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/true);
7382 break;
7383
7384 // TODO: bfloat conversions
7385 // - bfloat @llvm.aarch64.neon.bfcvt(float)
7386 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float>)
7387 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat>, <4 x float>)
7388
7389 // Add reduction to scalar
7390 case Intrinsic::aarch64_neon_faddv:
7391 case Intrinsic::aarch64_neon_saddv:
7392 case Intrinsic::aarch64_neon_uaddv:
7393 // Signed/Unsigned min/max (Vector)
7394 // TODO: handling similarly to AND/OR may be more precise.
7395 case Intrinsic::aarch64_neon_smaxv:
7396 case Intrinsic::aarch64_neon_sminv:
7397 case Intrinsic::aarch64_neon_umaxv:
7398 case Intrinsic::aarch64_neon_uminv:
7399 // Floating-point min/max (vector)
7400 // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
7401 // but our shadow propagation is the same.
7402 case Intrinsic::aarch64_neon_fmaxv:
7403 case Intrinsic::aarch64_neon_fminv:
7404 case Intrinsic::aarch64_neon_fmaxnmv:
7405 case Intrinsic::aarch64_neon_fminnmv:
7406 // Sum long across vector
7407 case Intrinsic::aarch64_neon_saddlv:
7408 case Intrinsic::aarch64_neon_uaddlv:
7409 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/true);
7410 break;
7411
7412 case Intrinsic::aarch64_neon_ld1x2:
7413 case Intrinsic::aarch64_neon_ld1x3:
7414 case Intrinsic::aarch64_neon_ld1x4:
7415 case Intrinsic::aarch64_neon_ld2:
7416 case Intrinsic::aarch64_neon_ld3:
7417 case Intrinsic::aarch64_neon_ld4:
7418 case Intrinsic::aarch64_neon_ld2r:
7419 case Intrinsic::aarch64_neon_ld3r:
7420 case Intrinsic::aarch64_neon_ld4r: {
7421 handleNEONVectorLoad(I, /*WithLane=*/false);
7422 break;
7423 }
7424
7425 case Intrinsic::aarch64_neon_ld2lane:
7426 case Intrinsic::aarch64_neon_ld3lane:
7427 case Intrinsic::aarch64_neon_ld4lane: {
7428 handleNEONVectorLoad(I, /*WithLane=*/true);
7429 break;
7430 }
7431
7432 // Saturating extract narrow
7433 case Intrinsic::aarch64_neon_sqxtn:
7434 case Intrinsic::aarch64_neon_sqxtun:
7435 case Intrinsic::aarch64_neon_uqxtn:
7436 // These only have one argument, but we (ab)use handleShadowOr because it
7437 // does work on single argument intrinsics and will typecast the shadow
7438 // (and update the origin).
7439 handleShadowOr(I);
7440 break;
7441
7442 case Intrinsic::aarch64_neon_st1x2:
7443 case Intrinsic::aarch64_neon_st1x3:
7444 case Intrinsic::aarch64_neon_st1x4:
7445 case Intrinsic::aarch64_neon_st2:
7446 case Intrinsic::aarch64_neon_st3:
7447 case Intrinsic::aarch64_neon_st4: {
7448 handleNEONVectorStoreIntrinsic(I, useLane: false);
7449 break;
7450 }
7451
7452 case Intrinsic::aarch64_neon_st2lane:
7453 case Intrinsic::aarch64_neon_st3lane:
7454 case Intrinsic::aarch64_neon_st4lane: {
7455 handleNEONVectorStoreIntrinsic(I, useLane: true);
7456 break;
7457 }
7458
7459 // Arm NEON vector table intrinsics have the source/table register(s) as
7460 // arguments, followed by the index register. They return the output.
7461 //
7462 // 'TBL writes a zero if an index is out-of-range, while TBX leaves the
7463 // original value unchanged in the destination register.'
7464 // Conveniently, zero denotes a clean shadow, which means out-of-range
7465 // indices for TBL will initialize the user data with zero and also clean
7466 // the shadow. (For TBX, neither the user data nor the shadow will be
7467 // updated, which is also correct.)
7468 case Intrinsic::aarch64_neon_tbl1:
7469 case Intrinsic::aarch64_neon_tbl2:
7470 case Intrinsic::aarch64_neon_tbl3:
7471 case Intrinsic::aarch64_neon_tbl4:
7472 case Intrinsic::aarch64_neon_tbx1:
7473 case Intrinsic::aarch64_neon_tbx2:
7474 case Intrinsic::aarch64_neon_tbx3:
7475 case Intrinsic::aarch64_neon_tbx4: {
7476 // The last trailing argument (index register) should be handled verbatim
7477 handleIntrinsicByApplyingToShadow(
7478 I, /*shadowIntrinsicID=*/I.getIntrinsicID(),
7479 /*trailingVerbatimArgs=*/1, /*forceIntegerIntrinsic=*/false);
7480 break;
7481 }
7482
7483 case Intrinsic::aarch64_neon_fmulx:
7484 case Intrinsic::aarch64_neon_pmul:
7485 case Intrinsic::aarch64_neon_pmull:
7486 case Intrinsic::aarch64_neon_smull:
7487 case Intrinsic::aarch64_neon_pmull64:
7488 case Intrinsic::aarch64_neon_umull: {
7489 handleNEONVectorMultiplyIntrinsic(I);
7490 break;
7491 }
7492
7493 case Intrinsic::aarch64_neon_smmla:
7494 case Intrinsic::aarch64_neon_ummla:
7495 case Intrinsic::aarch64_neon_usmmla:
7496 case Intrinsic::aarch64_neon_bfmmla:
7497 handleNEONMatrixMultiply(I);
7498 break;
7499
7500 // <2 x i32> @llvm.aarch64.neon.{u,s,us}dot.v2i32.v8i8
7501 // (<2 x i32> %acc, <8 x i8> %a, <8 x i8> %b)
7502 // <4 x i32> @llvm.aarch64.neon.{u,s,us}dot.v4i32.v16i8
7503 // (<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b)
7504 case Intrinsic::aarch64_neon_sdot:
7505 case Intrinsic::aarch64_neon_udot:
7506 case Intrinsic::aarch64_neon_usdot:
7507 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7508 /*ZeroPurifies=*/true,
7509 /*EltSizeInBits=*/0,
7510 /*Lanes=*/kBothLanes);
7511 break;
7512
7513 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
7514 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
7515 // <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16
7516 // (<4 x float> %acc, <8 x bfloat> %a, <8 x bfloat> %b)
7517 case Intrinsic::aarch64_neon_bfdot:
7518 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7519 /*ZeroPurifies=*/false,
7520 /*EltSizeInBits=*/0,
7521 /*Lanes=*/kBothLanes);
7522 break;
7523
7524 // <4 x half > @llvm.aarch64.neon.fp8.fdot2
7525 // (<4 x half >, < 8 x i8>, < 8 x i8>)
7526 // <8 x half > @llvm.aarch64.neon.fp8.fdot2
7527 // (<8 x half >, <16 x i8>, <16 x i8>)
7528 //
7529 // N.B. although the multiplicands are i8, they are actually fp8, thus
7530 // ZeroPurifies is not applicable.
7531 case Intrinsic::aarch64_neon_fp8_fdot2:
7532 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7533 /*ZeroPurifies=*/false,
7534 /*EltSizeInBits=*/0,
7535 /*Lanes=*/kBothLanes);
7536 break;
7537
7538 // <2 x float> @llvm.aarch64.neon.fp8.fdot4
7539 // (<2 x float>, < 8 x i8>, < 8 x i8>)
7540 // <4 x float> @llvm.aarch64.neon.fp8.fdot4
7541 // (<4 x float>, <16 x i8>, <16 x i8>)
7542 //
7543 // N.B. although the multiplicands are i8, they are actually fp8, thus
7544 // ZeroPurifies is not applicable.
7545 case Intrinsic::aarch64_neon_fp8_fdot4:
7546 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7547 /*ZeroPurifies=*/false,
7548 /*EltSizeInBits=*/0,
7549 /*Lanes=*/kBothLanes);
7550 break;
7551
7552 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane
7553 // (<4 x half>, <8 x i8>, <16 x i8>, i32)
7554 // <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane
7555 // (<8 x half>, <16 x i8>, <16 x i8>, i32)
7556 case Intrinsic::aarch64_neon_fp8_fdot2_lane:
7557 handleNEONDotProductLaneIntrinsic(I, /*ReductionFactor=*/2);
7558 break;
7559
7560 // <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane
7561 // (<2 x float>, <8 x i8>, <16 x i8>, i32)
7562 // <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane
7563 // (<4 x float>, <16 x i8>, <16 x i8>, i32)
7564 case Intrinsic::aarch64_neon_fp8_fdot4_lane:
7565 handleNEONDotProductLaneIntrinsic(I, /*ReductionFactor=*/4);
7566 break;
7567
7568 // Floating-Point Absolute Compare Greater Than/Equal
7569 case Intrinsic::aarch64_neon_facge:
7570 case Intrinsic::aarch64_neon_facgt:
7571 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/false);
7572 break;
7573
7574 default:
7575 return false;
7576 }
7577
7578 return true;
7579 }
7580
7581 void visitIntrinsicInst(IntrinsicInst &I) {
7582 if (maybeHandleCrossPlatformIntrinsic(I))
7583 return;
7584
7585 if (maybeHandleX86SIMDIntrinsic(I))
7586 return;
7587
7588 if (maybeHandleArmSIMDIntrinsic(I))
7589 return;
7590
7591 if (maybeHandleUnknownIntrinsic(I))
7592 return;
7593
7594 visitInstruction(I);
7595 }
7596
7597 void visitLibAtomicLoad(CallBase &CB) {
7598 // Since we use getNextNode here, we can't have CB terminate the BB.
7599 assert(isa<CallInst>(CB));
7600
7601 IRBuilder<> IRB(&CB);
7602 Value *Size = CB.getArgOperand(i: 0);
7603 Value *SrcPtr = CB.getArgOperand(i: 1);
7604 Value *DstPtr = CB.getArgOperand(i: 2);
7605 Value *Ordering = CB.getArgOperand(i: 3);
7606 // Convert the call to have at least Acquire ordering to make sure
7607 // the shadow operations aren't reordered before it.
7608 Value *NewOrdering =
7609 IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
7610 CB.setArgOperand(i: 3, v: NewOrdering);
7611
7612 NextNodeIRBuilder NextIRB(&CB);
7613 Value *SrcShadowPtr, *SrcOriginPtr;
7614 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
7615 getShadowOriginPtr(Addr: SrcPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7616 /*isStore*/ false);
7617 Value *DstShadowPtr =
7618 getShadowOriginPtr(Addr: DstPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7619 /*isStore*/ true)
7620 .first;
7621
7622 NextIRB.CreateMemCpy(Dst: DstShadowPtr, DstAlign: Align(1), Src: SrcShadowPtr, SrcAlign: Align(1), Size);
7623 if (MS.TrackOrigins) {
7624 Value *SrcOrigin = NextIRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr,
7625 Align: kMinOriginAlignment);
7626 Value *NewOrigin = updateOrigin(V: SrcOrigin, IRB&: NextIRB);
7627 NextIRB.CreateCall(Callee: MS.MsanSetOriginFn, Args: {DstPtr, Size, NewOrigin});
7628 }
7629 }
7630
7631 void visitLibAtomicStore(CallBase &CB) {
7632 IRBuilder<> IRB(&CB);
7633 Value *Size = CB.getArgOperand(i: 0);
7634 Value *DstPtr = CB.getArgOperand(i: 2);
7635 Value *Ordering = CB.getArgOperand(i: 3);
7636 // Convert the call to have at least Release ordering to make sure
7637 // the shadow operations aren't reordered after it.
7638 Value *NewOrdering =
7639 IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
7640 CB.setArgOperand(i: 3, v: NewOrdering);
7641
7642 Value *DstShadowPtr =
7643 getShadowOriginPtr(Addr: DstPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1),
7644 /*isStore*/ true)
7645 .first;
7646
7647 // Atomic store always paints clean shadow/origin. See file header.
7648 IRB.CreateMemSet(Ptr: DstShadowPtr, Val: getCleanShadow(OrigTy: IRB.getInt8Ty()), Size,
7649 Align: Align(1));
7650 }
7651
7652 void visitCallBase(CallBase &CB) {
7653 assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
7654 if (CB.isInlineAsm()) {
7655 // For inline asm (either a call to asm function, or callbr instruction),
7656 // do the usual thing: check argument shadow and mark all outputs as
7657 // clean. Note that any side effects of the inline asm that are not
7658 // immediately visible in its constraints are not handled.
7659 if (Opts.msan_handle_asm_conservative)
7660 visitAsmInstruction(I&: CB);
7661 else
7662 visitInstruction(I&: CB);
7663 return;
7664 }
7665 LibFunc LF = TLI->getLibFunc(CB);
7666 if (LF != NotLibFunc) {
7667 // libatomic.a functions need to have special handling because there isn't
7668 // a good way to intercept them or compile the library with
7669 // instrumentation.
7670 switch (LF) {
7671 case LibFunc_atomic_load:
7672 if (!isa<CallInst>(Val: CB)) {
7673 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
7674 "Ignoring!\n";
7675 break;
7676 }
7677 visitLibAtomicLoad(CB);
7678 return;
7679 case LibFunc_atomic_store:
7680 visitLibAtomicStore(CB);
7681 return;
7682 default:
7683 break;
7684 }
7685 }
7686
7687 if (auto *Call = dyn_cast<CallInst>(Val: &CB)) {
7688 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
7689
7690 // We are going to insert code that relies on the fact that the callee
7691 // will become a non-readonly function after it is instrumented by us. To
7692 // prevent this code from being optimized out, mark that function
7693 // non-readonly in advance.
7694 // TODO: We can likely do better than dropping memory() completely here.
7695 AttributeMask B;
7696 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
7697
7698 Call->removeFnAttrs(AttrsToRemove: B);
7699 if (Function *Func = Call->getCalledFunction()) {
7700 Func->removeFnAttrs(Attrs: B);
7701 }
7702
7703 maybeMarkSanitizerLibraryCallNoBuiltin(CI: Call, TLI);
7704 }
7705 IRBuilder<> IRB(&CB);
7706 bool MayCheckCall = MS.EagerChecks;
7707 if (Function *Func = CB.getCalledFunction()) {
7708 // __sanitizer_unaligned_{load,store} functions may be called by users
7709 // and always expects shadows in the TLS. So don't check them.
7710 MayCheckCall &= !Func->getName().starts_with(Prefix: "__sanitizer_unaligned_");
7711 }
7712
7713 unsigned ArgOffset = 0;
7714 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
7715 for (const auto &[i, A] : llvm::enumerate(First: CB.args())) {
7716 if (!A->getType()->isSized()) {
7717 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
7718 continue;
7719 }
7720
7721 if (A->getType()->isScalableTy()) {
7722 LLVM_DEBUG(dbgs() << "Arg " << i << " is vscale: " << CB << "\n");
7723 // Handle as noundef, but don't reserve tls slots.
7724 insertCheckShadowOf(Val: A, OrigIns: &CB);
7725 continue;
7726 }
7727
7728 unsigned Size = 0;
7729 const DataLayout &DL = F.getDataLayout();
7730
7731 bool ByVal = CB.isByValArgument(ArgNo: i);
7732 bool NoUndef = CB.paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef);
7733 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
7734
7735 if (EagerCheck) {
7736 insertCheckShadowOf(Val: A, OrigIns: &CB);
7737 Size = DL.getTypeAllocSize(Ty: A->getType());
7738 } else {
7739 [[maybe_unused]] Value *Store = nullptr;
7740 // Compute the Shadow for arg even if it is ByVal, because
7741 // in that case getShadow() will copy the actual arg shadow to
7742 // __msan_param_tls.
7743 Value *ArgShadow = getShadow(V: A);
7744 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
7745 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
7746 << " Shadow: " << *ArgShadow << "\n");
7747 if (ByVal) {
7748 // ByVal requires some special handling as it's too big for a single
7749 // load
7750 assert(A->getType()->isPointerTy() &&
7751 "ByVal argument is not a pointer!");
7752 Size = DL.getTypeAllocSize(Ty: CB.getParamByValType(ArgNo: i));
7753 if (ArgOffset + Size > kParamTLSSize)
7754 break;
7755 const MaybeAlign ParamAlignment(CB.getParamAlign(ArgNo: i));
7756 MaybeAlign Alignment = std::nullopt;
7757 if (ParamAlignment)
7758 Alignment = std::min(a: *ParamAlignment, b: kShadowTLSAlignment);
7759 Value *AShadowPtr, *AOriginPtr;
7760 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
7761 getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
7762 /*isStore*/ false);
7763 if (!PropagateShadow) {
7764 Store = IRB.CreateMemSet(Ptr: ArgShadowBase,
7765 Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
7766 Size, Align: Alignment);
7767 } else {
7768 Store = IRB.CreateMemCpy(Dst: ArgShadowBase, DstAlign: Alignment, Src: AShadowPtr,
7769 SrcAlign: Alignment, Size);
7770 if (MS.TrackOrigins) {
7771 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
7772 // FIXME: OriginSize should be:
7773 // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
7774 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
7775 IRB.CreateMemCpy(
7776 Dst: ArgOriginBase,
7777 /* by origin_tls[ArgOffset] */ DstAlign: kMinOriginAlignment,
7778 Src: AOriginPtr,
7779 /* by getShadowOriginPtr */ SrcAlign: kMinOriginAlignment, Size: OriginSize);
7780 }
7781 }
7782 } else {
7783 // Any other parameters mean we need bit-grained tracking of uninit
7784 // data
7785 Size = DL.getTypeAllocSize(Ty: A->getType());
7786 if (ArgOffset + Size > kParamTLSSize)
7787 break;
7788 Store = IRB.CreateAlignedStore(Val: ArgShadow, Ptr: ArgShadowBase,
7789 Align: kShadowTLSAlignment);
7790 Constant *Cst = dyn_cast<Constant>(Val: ArgShadow);
7791 if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
7792 IRB.CreateStore(Val: getOrigin(V: A),
7793 Ptr: getOriginPtrForArgument(IRB, ArgOffset));
7794 }
7795 }
7796 assert(Store != nullptr);
7797 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
7798 }
7799 assert(Size != 0);
7800 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
7801 }
7802 LLVM_DEBUG(dbgs() << " done with call args\n");
7803
7804 FunctionType *FT = CB.getFunctionType();
7805 if (FT->isVarArg()) {
7806 VAHelper->visitCallBase(CB, IRB);
7807 }
7808
7809 // Now, get the shadow for the RetVal.
7810 if (!CB.getType()->isSized())
7811 return;
7812 // Don't emit the epilogue for musttail call returns.
7813 if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
7814 return;
7815
7816 if (MayCheckCall && CB.hasRetAttr(Kind: Attribute::NoUndef)) {
7817 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7818 setOrigin(V: &CB, Origin: getCleanOrigin());
7819 return;
7820 }
7821
7822 IRBuilder<> IRBBefore(&CB);
7823 // Until we have full dynamic coverage, make sure the retval shadow is 0.
7824 Value *Base = getShadowPtrForRetval(IRB&: IRBBefore);
7825 IRBBefore.CreateAlignedStore(Val: getCleanShadow(V: &CB), Ptr: Base,
7826 Align: kShadowTLSAlignment);
7827 BasicBlock::iterator NextInsn;
7828 if (isa<CallInst>(Val: CB)) {
7829 NextInsn = ++CB.getIterator();
7830 assert(NextInsn != CB.getParent()->end());
7831 } else {
7832 BasicBlock *NormalDest = cast<InvokeInst>(Val&: CB).getNormalDest();
7833 if (!NormalDest->getSinglePredecessor()) {
7834 // FIXME: this case is tricky, so we are just conservative here.
7835 // Perhaps we need to split the edge between this BB and NormalDest,
7836 // but a naive attempt to use SplitEdge leads to a crash.
7837 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7838 setOrigin(V: &CB, Origin: getCleanOrigin());
7839 return;
7840 }
7841 // FIXME: NextInsn is likely in a basic block that has not been visited
7842 // yet. Anything inserted there will be instrumented by MSan later!
7843 NextInsn = NormalDest->getFirstInsertionPt();
7844 assert(NextInsn != NormalDest->end() &&
7845 "Could not find insertion point for retval shadow load");
7846 }
7847 IRBuilder<> IRBAfter(&*NextInsn);
7848 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
7849 Ty: getShadowTy(V: &CB), Ptr: getShadowPtrForRetval(IRB&: IRBAfter), Align: kShadowTLSAlignment,
7850 Name: "_msret");
7851 setShadow(V: &CB, SV: RetvalShadow);
7852 if (MS.TrackOrigins)
7853 setOrigin(V: &CB, Origin: IRBAfter.CreateLoad(Ty: MS.OriginTy, Ptr: getOriginPtrForRetval()));
7854 }
7855
7856 bool isAMustTailRetVal(Value *RetVal) {
7857 if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
7858 RetVal = I->getOperand(i_nocapture: 0);
7859 }
7860 if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
7861 return I->isMustTailCall();
7862 }
7863 return false;
7864 }
7865
7866 void visitReturnInst(ReturnInst &I) {
7867 IRBuilder<> IRB(&I);
7868 Value *RetVal = I.getReturnValue();
7869 if (!RetVal)
7870 return;
7871 // Don't emit the epilogue for musttail call returns.
7872 if (isAMustTailRetVal(RetVal))
7873 return;
7874 Value *ShadowPtr = getShadowPtrForRetval(IRB);
7875 bool HasNoUndef = F.hasRetAttribute(Kind: Attribute::NoUndef);
7876 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
7877 // FIXME: Consider using SpecialCaseList to specify a list of functions that
7878 // must always return fully initialized values. For now, we hardcode "main".
7879 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
7880
7881 Value *Shadow = getShadow(V: RetVal);
7882 bool StoreOrigin = true;
7883 if (EagerCheck) {
7884 insertCheckShadowOf(Val: RetVal, OrigIns: &I);
7885 Shadow = getCleanShadow(V: RetVal);
7886 StoreOrigin = false;
7887 }
7888
7889 // The caller may still expect information passed over TLS if we pass our
7890 // check
7891 if (StoreShadow) {
7892 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: kShadowTLSAlignment);
7893 if (MS.TrackOrigins && StoreOrigin)
7894 IRB.CreateStore(Val: getOrigin(V: RetVal), Ptr: getOriginPtrForRetval());
7895 }
7896 }
7897
7898 void visitPHINode(PHINode &I) {
7899 IRBuilder<> IRB(&I);
7900 if (!PropagateShadow) {
7901 setShadow(V: &I, SV: getCleanShadow(V: &I));
7902 setOrigin(V: &I, Origin: getCleanOrigin());
7903 return;
7904 }
7905
7906 ShadowPHINodes.push_back(Elt: &I);
7907 setShadow(V: &I, SV: IRB.CreatePHI(Ty: getShadowTy(V: &I), NumReservedValues: I.getNumIncomingValues(),
7908 Name: "_msphi_s"));
7909 if (MS.TrackOrigins)
7910 setOrigin(
7911 V: &I, Origin: IRB.CreatePHI(Ty: MS.OriginTy, NumReservedValues: I.getNumIncomingValues(), Name: "_msphi_o"));
7912 }
7913
7914 Value *getLocalVarIdptr(AllocaInst &I) {
7915 ConstantInt *IntConst =
7916 ConstantInt::get(Ty: Type::getInt32Ty(C&: (*F.getParent()).getContext()), V: 0);
7917 return new GlobalVariable(*F.getParent(), IntConst->getType(),
7918 /*isConstant=*/false, GlobalValue::PrivateLinkage,
7919 IntConst);
7920 }
7921
7922 Value *getLocalVarDescription(AllocaInst &I) {
7923 return createPrivateConstGlobalForString(M&: *F.getParent(), Str: I.getName());
7924 }
7925
7926 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7927 if (PoisonStack && Opts.msan_poison_stack_with_call) {
7928 IRB.CreateCall(Callee: MS.MsanPoisonStackFn, Args: {&I, Len});
7929 } else {
7930 Value *ShadowBase, *OriginBase;
7931 std::tie(args&: ShadowBase, args&: OriginBase) = getShadowOriginPtr(
7932 Addr: &I, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1), /*isStore*/ true);
7933
7934 Value *PoisonValue =
7935 IRB.getInt8(C: PoisonStack ? Opts.msan_poison_stack_pattern : 0);
7936 IRB.CreateMemSet(Ptr: ShadowBase, Val: PoisonValue, Size: Len, Align: I.getAlign());
7937 }
7938
7939 if (PoisonStack && MS.TrackOrigins) {
7940 Value *Idptr = getLocalVarIdptr(I);
7941 if (Opts.msan_print_stack_names) {
7942 Value *Descr = getLocalVarDescription(I);
7943 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginWithDescriptionFn,
7944 Args: {&I, Len, Idptr, Descr});
7945 } else {
7946 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginNoDescriptionFn, Args: {&I, Len, Idptr});
7947 }
7948 }
7949 }
7950
7951 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7952 Value *Descr = getLocalVarDescription(I);
7953 if (PoisonStack) {
7954 IRB.CreateCall(Callee: MS.MsanPoisonAllocaFn, Args: {&I, Len, Descr});
7955 } else {
7956 IRB.CreateCall(Callee: MS.MsanUnpoisonAllocaFn, Args: {&I, Len});
7957 }
7958 }
7959
7960 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
7961 if (!InsPoint)
7962 InsPoint = &I;
7963 NextNodeIRBuilder IRB(InsPoint);
7964 Value *Len = IRB.CreateAllocationSize(DestTy: MS.IntptrTy, AI: &I);
7965
7966 if (MS.CompileKernel)
7967 poisonAllocaKmsan(I, IRB, Len);
7968 else
7969 poisonAllocaUserspace(I, IRB, Len);
7970 }
7971
7972 void visitAllocaInst(AllocaInst &I) {
7973 setShadow(V: &I, SV: getCleanShadow(V: &I));
7974 setOrigin(V: &I, Origin: getCleanOrigin());
7975 // We'll get to this alloca later unless it's poisoned at the corresponding
7976 // llvm.lifetime.start.
7977 AllocaSet.insert(X: &I);
7978 }
7979
7980 void visitSelectInst(SelectInst &I) {
7981 // a = select b, c, d
7982 Value *B = I.getCondition();
7983 Value *C = I.getTrueValue();
7984 Value *D = I.getFalseValue();
7985
7986 handleSelectLikeInst(I, B, C, D);
7987 }
7988
7989 void handleSelectLikeInst(Instruction &I, Value *B, Value *C, Value *D) {
7990 IRBuilder<> IRB(&I);
7991
7992 Value *Sb = getShadow(V: B);
7993 Value *Sc = getShadow(V: C);
7994 Value *Sd = getShadow(V: D);
7995
7996 Value *Ob = MS.TrackOrigins ? getOrigin(V: B) : nullptr;
7997 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
7998 Value *Od = MS.TrackOrigins ? getOrigin(V: D) : nullptr;
7999
8000 // Result shadow if condition shadow is 0.
8001 Value *Sa0 = IRB.CreateSelect(C: B, True: Sc, False: Sd);
8002 Value *Sa1;
8003 if (I.getType()->isAggregateType()) {
8004 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
8005 // an extra "select". This results in much more compact IR.
8006 // Sa = select Sb, poisoned, (select b, Sc, Sd)
8007 Sa1 = getPoisonedShadow(ShadowTy: getShadowTy(OrigTy: I.getType()));
8008 } else if (isScalableNonVectorType(Ty: I.getType())) {
8009 // This is intended to handle target("aarch64.svcount"), which can't be
8010 // handled in the else branch because of incompatibility with CreateXor
8011 // ("The supported LLVM operations on this type are limited to load,
8012 // store, phi, select and alloca instructions").
8013
8014 // TODO: this currently underapproximates. Use Arm SVE EOR in the else
8015 // branch as needed instead.
8016 Sa1 = getCleanShadow(OrigTy: getShadowTy(OrigTy: I.getType()));
8017 } else {
8018 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
8019 // If Sb (condition is poisoned), look for bits in c and d that are equal
8020 // and both unpoisoned.
8021 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
8022
8023 // Cast arguments to shadow-compatible type.
8024 C = CreateAppToShadowCast(IRB, V: C);
8025 D = CreateAppToShadowCast(IRB, V: D);
8026
8027 // Result shadow if condition shadow is 1.
8028 Sa1 = IRB.CreateOr(Ops: {IRB.CreateXor(LHS: C, RHS: D), Sc, Sd});
8029 }
8030 Value *Sa = IRB.CreateSelect(C: Sb, True: Sa1, False: Sa0, Name: "_msprop_select");
8031 setShadow(V: &I, SV: Sa);
8032 if (MS.TrackOrigins) {
8033 // Origins are always i32, so any vector conditions must be flattened.
8034 // FIXME: consider tracking vector origins for app vectors?
8035 if (B->getType()->isVectorTy()) {
8036 B = convertToBool(V: B, IRB);
8037 Sb = convertToBool(V: Sb, IRB);
8038 }
8039 // a = select b, c, d
8040 // Oa = Sb ? Ob : (b ? Oc : Od)
8041 setOrigin(V: &I, Origin: IRB.CreateSelect(C: Sb, True: Ob, False: IRB.CreateSelect(C: B, True: Oc, False: Od)));
8042 }
8043 }
8044
8045 void visitLandingPadInst(LandingPadInst &I) {
8046 // Do nothing.
8047 // See https://github.com/google/sanitizers/issues/504
8048 setShadow(V: &I, SV: getCleanShadow(V: &I));
8049 setOrigin(V: &I, Origin: getCleanOrigin());
8050 }
8051
8052 void visitCatchSwitchInst(CatchSwitchInst &I) {
8053 setShadow(V: &I, SV: getCleanShadow(V: &I));
8054 setOrigin(V: &I, Origin: getCleanOrigin());
8055 }
8056
8057 void visitFuncletPadInst(FuncletPadInst &I) {
8058 setShadow(V: &I, SV: getCleanShadow(V: &I));
8059 setOrigin(V: &I, Origin: getCleanOrigin());
8060 }
8061
8062 void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
8063
8064 void visitExtractValueInst(ExtractValueInst &I) {
8065 IRBuilder<> IRB(&I);
8066 Value *Agg = I.getAggregateOperand();
8067 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
8068 Value *AggShadow = getShadow(V: Agg);
8069 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
8070 Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
8071 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
8072 setShadow(V: &I, SV: ResShadow);
8073 setOriginForNaryOp(I);
8074 }
8075
8076 void visitInsertValueInst(InsertValueInst &I) {
8077 IRBuilder<> IRB(&I);
8078 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
8079 Value *AggShadow = getShadow(V: I.getAggregateOperand());
8080 Value *InsShadow = getShadow(V: I.getInsertedValueOperand());
8081 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
8082 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
8083 Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
8084 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
8085 setShadow(V: &I, SV: Res);
8086 setOriginForNaryOp(I);
8087 }
8088
8089 void dumpInst(Instruction &I, const Twine &Prefix) {
8090 // Instruction name only
8091 // For intrinsics, the full/overloaded name is used
8092 //
8093 // e.g., "call llvm.aarch64.neon.uqsub.v16i8"
8094 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
8095 errs() << "ZZZ:" << Prefix << " call "
8096 << CI->getCalledFunction()->getName() << "\n";
8097 } else {
8098 errs() << "ZZZ:" << Prefix << " " << I.getOpcodeName() << "\n";
8099 }
8100
8101 // Instruction prototype (including return type and parameter types)
8102 // For intrinsics, we use the base/non-overloaded name
8103 //
8104 // e.g., "call <16 x i8> @llvm.aarch64.neon.uqsub(<16 x i8>, <16 x i8>)"
8105 unsigned NumOperands = I.getNumOperands();
8106 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
8107 errs() << "YYY:" << Prefix << " call " << *I.getType() << " @";
8108
8109 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI))
8110 errs() << Intrinsic::getBaseName(id: II->getIntrinsicID());
8111 else
8112 errs() << CI->getCalledFunction()->getName();
8113
8114 errs() << "(";
8115
8116 // The last operand of a CallInst is the function itself.
8117 NumOperands--;
8118 } else
8119 errs() << "YYY:" << Prefix << " " << *I.getType() << " "
8120 << I.getOpcodeName() << "(";
8121
8122 for (size_t i = 0; i < NumOperands; i++) {
8123 if (i > 0)
8124 errs() << ", ";
8125
8126 errs() << *(I.getOperand(i)->getType());
8127 }
8128
8129 errs() << ")\n";
8130
8131 // Full instruction, including types and operand values
8132 // For intrinsics, the full/overloaded name is used
8133 //
8134 // e.g., "%vqsubq_v.i15 = call noundef <16 x i8>
8135 // @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %vext21.i,
8136 // <16 x i8> splat (i8 1)), !dbg !66"
8137 errs() << "QQQ:" << Prefix << " " << I << "\n";
8138 }
8139
8140 void visitResumeInst(ResumeInst &I) {
8141 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
8142 // Nothing to do here.
8143 }
8144
8145 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
8146 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
8147 // Nothing to do here.
8148 }
8149
8150 void visitCatchReturnInst(CatchReturnInst &CRI) {
8151 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
8152 // Nothing to do here.
8153 }
8154
8155 void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
8156 IRBuilder<> &IRB, const DataLayout &DL,
8157 bool isOutput) {
8158 // For each assembly argument, we check its value for being initialized.
8159 // If the argument is a pointer, we assume it points to a single element
8160 // of the corresponding type (or to a 8-byte word, if the type is unsized).
8161 // Each such pointer is instrumented with a call to the runtime library.
8162 Type *OpType = Operand->getType();
8163 // Check the operand value itself.
8164 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8165 if (!OpType->isPointerTy() || !isOutput) {
8166 assert(!isOutput);
8167 return;
8168 }
8169 if (!ElemTy->isSized())
8170 return;
8171 auto Size = DL.getTypeStoreSize(Ty: ElemTy);
8172 Value *SizeVal = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size);
8173 if (MS.CompileKernel) {
8174 IRB.CreateCall(Callee: MS.MsanInstrumentAsmStoreFn, Args: {Operand, SizeVal});
8175 } else {
8176 // ElemTy, derived from elementtype(), does not encode the alignment of
8177 // the pointer. Conservatively assume that the shadow memory is unaligned.
8178 // When Size is large, avoid StoreInst as it would expand to many
8179 // instructions.
8180 auto [ShadowPtr, _] =
8181 getShadowOriginPtrUserspace(Addr: Operand, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1));
8182 if (Size <= 32)
8183 IRB.CreateAlignedStore(Val: getCleanShadow(OrigTy: ElemTy), Ptr: ShadowPtr, Align: Align(1));
8184 else
8185 IRB.CreateMemSet(Ptr: ShadowPtr, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8186 Size: SizeVal, Align: Align(1));
8187 }
8188 }
8189
8190 /// Get the number of output arguments returned by pointers.
8191 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
8192 int NumRetOutputs = 0;
8193 int NumOutputs = 0;
8194 Type *RetTy = cast<Value>(Val: CB)->getType();
8195 if (!RetTy->isVoidTy()) {
8196 // Register outputs are returned via the CallInst return value.
8197 auto *ST = dyn_cast<StructType>(Val: RetTy);
8198 if (ST)
8199 NumRetOutputs = ST->getNumElements();
8200 else
8201 NumRetOutputs = 1;
8202 }
8203 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
8204 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
8205 switch (Info.Type) {
8206 case InlineAsm::isOutput:
8207 NumOutputs++;
8208 break;
8209 default:
8210 break;
8211 }
8212 }
8213 return NumOutputs - NumRetOutputs;
8214 }
8215
8216 void visitAsmInstruction(Instruction &I) {
8217 // Conservative inline assembly handling: check for poisoned shadow of
8218 // asm() arguments, then unpoison the result and all the memory locations
8219 // pointed to by those arguments.
8220 // An inline asm() statement in C++ contains lists of input and output
8221 // arguments used by the assembly code. These are mapped to operands of the
8222 // CallInst as follows:
8223 // - nR register outputs ("=r) are returned by value in a single structure
8224 // (SSA value of the CallInst);
8225 // - nO other outputs ("=m" and others) are returned by pointer as first
8226 // nO operands of the CallInst;
8227 // - nI inputs ("r", "m" and others) are passed to CallInst as the
8228 // remaining nI operands.
8229 // The total number of asm() arguments in the source is nR+nO+nI, and the
8230 // corresponding CallInst has nO+nI+1 operands (the last operand is the
8231 // function to be called).
8232 const DataLayout &DL = F.getDataLayout();
8233 CallBase *CB = cast<CallBase>(Val: &I);
8234 IRBuilder<> IRB(&I);
8235 InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8236 int OutputArgs = getNumOutputArgs(IA, CB);
8237 // The last operand of a CallInst is the function itself.
8238 int NumOperands = CB->getNumOperands() - 1;
8239
8240 // Check input arguments. Doing so before unpoisoning output arguments, so
8241 // that we won't overwrite uninit values before checking them.
8242 for (int i = OutputArgs; i < NumOperands; i++) {
8243 Value *Operand = CB->getOperand(i_nocapture: i);
8244 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8245 /*isOutput*/ false);
8246 }
8247 // Unpoison output arguments. This must happen before the actual InlineAsm
8248 // call, so that the shadow for memory published in the asm() statement
8249 // remains valid.
8250 for (int i = 0; i < OutputArgs; i++) {
8251 Value *Operand = CB->getOperand(i_nocapture: i);
8252 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8253 /*isOutput*/ true);
8254 }
8255
8256 setShadow(V: &I, SV: getCleanShadow(V: &I));
8257 setOrigin(V: &I, Origin: getCleanOrigin());
8258 }
8259
8260 void visitFreezeInst(FreezeInst &I) {
8261 // Freeze always returns a fully defined value.
8262 setShadow(V: &I, SV: getCleanShadow(V: &I));
8263 setOrigin(V: &I, Origin: getCleanOrigin());
8264 }
8265
8266 void visitInstruction(Instruction &I) {
8267 // Everything else: stop propagating and check for poisoned shadow.
8268 if (Opts.msan_dump_strict_instructions)
8269 dumpInst(I, Prefix: "Strict");
8270 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
8271 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
8272 Value *Operand = I.getOperand(i);
8273 if (Operand->getType()->isSized())
8274 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8275 }
8276 setShadow(V: &I, SV: getCleanShadow(V: &I));
8277 setOrigin(V: &I, Origin: getCleanOrigin());
8278 }
8279};
8280
8281struct VarArgHelperBase : public VarArgHelper {
8282 Function &F;
8283 MemorySanitizer &MS;
8284 MemorySanitizerVisitor &MSV;
8285 SmallVector<CallInst *, 16> VAStartInstrumentationList;
8286 const unsigned VAListTagSize;
8287
8288 VarArgHelperBase(Function &F, MemorySanitizer &MS,
8289 MemorySanitizerVisitor &MSV, unsigned VAListTagSize)
8290 : F(F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
8291
8292 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8293 Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
8294 return IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
8295 }
8296
8297 /// Compute the shadow address for a given va_arg.
8298 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8299 return IRB.CreatePtrAdd(
8300 Ptr: MS.VAArgTLS, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset), Name: "_msarg_va_s");
8301 }
8302
8303 /// Compute the shadow address for a given va_arg.
8304 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset,
8305 unsigned ArgSize) {
8306 // Make sure we don't overflow __msan_va_arg_tls.
8307 if (ArgOffset + ArgSize > kParamTLSSize)
8308 return nullptr;
8309 return getShadowPtrForVAArgument(IRB, ArgOffset);
8310 }
8311
8312 /// Compute the origin address for a given va_arg.
8313 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
8314 // getOriginPtrForVAArgument() is always called after
8315 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
8316 // overflow.
8317 return IRB.CreatePtrAdd(Ptr: MS.VAArgOriginTLS,
8318 Offset: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset),
8319 Name: "_msarg_va_o");
8320 }
8321
8322 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
8323 unsigned BaseOffset) {
8324 // The tails of __msan_va_arg_tls is not large enough to fit full
8325 // value shadow, but it will be copied to backup anyway. Make it
8326 // clean.
8327 if (BaseOffset >= kParamTLSSize)
8328 return;
8329 Value *TailSize =
8330 ConstantInt::getSigned(Ty: IRB.getInt32Ty(), V: kParamTLSSize - BaseOffset);
8331 IRB.CreateMemSet(Ptr: ShadowBase, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8332 Size: TailSize, Align: Align(8));
8333 }
8334
8335 void unpoisonVAListTagForInst(IntrinsicInst &I) {
8336 IRBuilder<> IRB(&I);
8337 Value *VAListTag = I.getArgOperand(i: 0);
8338 const Align Alignment = Align(8);
8339 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
8340 Addr: VAListTag, IRB, ShadowTy: IRB.getInt8Ty(), Alignment, /*isStore*/ true);
8341 // Unpoison the whole __va_list_tag.
8342 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8343 Size: VAListTagSize, Align: Alignment, isVolatile: false);
8344 }
8345
8346 void visitVAStartInst(VAStartInst &I) override {
8347 if (F.getCallingConv() == CallingConv::Win64)
8348 return;
8349 VAStartInstrumentationList.push_back(Elt: &I);
8350 unpoisonVAListTagForInst(I);
8351 }
8352
8353 void visitVACopyInst(VACopyInst &I) override {
8354 if (F.getCallingConv() == CallingConv::Win64)
8355 return;
8356 unpoisonVAListTagForInst(I);
8357 }
8358};
8359
8360/// AMD64-specific implementation of VarArgHelper.
8361struct VarArgAMD64Helper : public VarArgHelperBase {
8362 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
8363 // See a comment in visitCallBase for more details.
8364 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
8365 static const unsigned AMD64FpEndOffsetSSE = 176;
8366 // If SSE is disabled, fp_offset in va_list is zero.
8367 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
8368
8369 unsigned AMD64FpEndOffset;
8370 AllocaInst *VAArgTLSCopy = nullptr;
8371 AllocaInst *VAArgTLSOriginCopy = nullptr;
8372 Value *VAArgOverflowSize = nullptr;
8373
8374 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8375
8376 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
8377 MemorySanitizerVisitor &MSV)
8378 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/24) {
8379 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
8380 for (const auto &Attr : F.getAttributes().getFnAttrs()) {
8381 if (Attr.isStringAttribute() &&
8382 (Attr.getKindAsString() == "target-features")) {
8383 if (Attr.getValueAsString().contains(Other: "-sse"))
8384 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
8385 break;
8386 }
8387 }
8388 }
8389
8390 ArgKind classifyArgument(Value *arg) {
8391 // A very rough approximation of X86_64 argument classification rules.
8392 Type *T = arg->getType();
8393 if (T->isX86_FP80Ty())
8394 return AK_Memory;
8395 if (T->isFPOrFPVectorTy())
8396 return AK_FloatingPoint;
8397 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
8398 return AK_GeneralPurpose;
8399 if (T->isPointerTy())
8400 return AK_GeneralPurpose;
8401 return AK_Memory;
8402 }
8403
8404 // For VarArg functions, store the argument shadow in an ABI-specific format
8405 // that corresponds to va_list layout.
8406 // We do this because Clang lowers va_arg in the frontend, and this pass
8407 // only sees the low level code that deals with va_list internals.
8408 // A much easier alternative (provided that Clang emits va_arg instructions)
8409 // would have been to associate each live instance of va_list with a copy of
8410 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
8411 // order.
8412 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8413 unsigned GpOffset = 0;
8414 unsigned FpOffset = AMD64GpEndOffset;
8415 unsigned OverflowOffset = AMD64FpEndOffset;
8416 const DataLayout &DL = F.getDataLayout();
8417
8418 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8419 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8420 bool IsByVal = CB.isByValArgument(ArgNo);
8421 if (IsByVal) {
8422 // ByVal arguments always go to the overflow area.
8423 // Fixed arguments passed through the overflow area will be stepped
8424 // over by va_start, so don't count them towards the offset.
8425 if (IsFixed)
8426 continue;
8427 assert(A->getType()->isPointerTy());
8428 Type *RealTy = CB.getParamByValType(ArgNo);
8429 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8430 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8431 unsigned BaseOffset = OverflowOffset;
8432 Value *ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8433 Value *OriginBase = nullptr;
8434 if (MS.TrackOrigins)
8435 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8436 OverflowOffset += AlignedSize;
8437
8438 if (OverflowOffset > kParamTLSSize) {
8439 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8440 continue; // We have no space to copy shadow there.
8441 }
8442
8443 Value *ShadowPtr, *OriginPtr;
8444 std::tie(args&: ShadowPtr, args&: OriginPtr) =
8445 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: kShadowTLSAlignment,
8446 /*isStore*/ false);
8447 IRB.CreateMemCpy(Dst: ShadowBase, DstAlign: kShadowTLSAlignment, Src: ShadowPtr,
8448 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8449 if (MS.TrackOrigins)
8450 IRB.CreateMemCpy(Dst: OriginBase, DstAlign: kShadowTLSAlignment, Src: OriginPtr,
8451 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8452 } else {
8453 ArgKind AK = classifyArgument(arg: A);
8454 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
8455 AK = AK_Memory;
8456 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
8457 AK = AK_Memory;
8458 Value *ShadowBase, *OriginBase = nullptr;
8459 switch (AK) {
8460 case AK_GeneralPurpose:
8461 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: GpOffset);
8462 if (MS.TrackOrigins)
8463 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset);
8464 GpOffset += 8;
8465 assert(GpOffset <= kParamTLSSize);
8466 break;
8467 case AK_FloatingPoint:
8468 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: FpOffset);
8469 if (MS.TrackOrigins)
8470 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
8471 FpOffset += 16;
8472 assert(FpOffset <= kParamTLSSize);
8473 break;
8474 case AK_Memory:
8475 if (IsFixed)
8476 continue;
8477 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8478 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8479 unsigned BaseOffset = OverflowOffset;
8480 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8481 if (MS.TrackOrigins) {
8482 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8483 }
8484 OverflowOffset += AlignedSize;
8485 if (OverflowOffset > kParamTLSSize) {
8486 // We have no space to copy shadow there.
8487 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8488 continue;
8489 }
8490 }
8491 // Take fixed arguments into account for GpOffset and FpOffset,
8492 // but don't actually store shadows for them.
8493 // TODO(glider): don't call get*PtrForVAArgument() for them.
8494 if (IsFixed)
8495 continue;
8496 Value *Shadow = MSV.getShadow(V: A);
8497 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowBase, Align: kShadowTLSAlignment);
8498 if (MS.TrackOrigins) {
8499 Value *Origin = MSV.getOrigin(V: A);
8500 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
8501 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
8502 Alignment: std::max(a: kShadowTLSAlignment, b: kMinOriginAlignment));
8503 }
8504 }
8505 }
8506 Constant *OverflowSize =
8507 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AMD64FpEndOffset);
8508 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8509 }
8510
8511 void finalizeInstrumentation() override {
8512 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8513 "finalizeInstrumentation called twice");
8514 if (!VAStartInstrumentationList.empty()) {
8515 // If there is a va_start in this function, make a backup copy of
8516 // va_arg_tls somewhere in the function entry block.
8517 IRBuilder<> IRB(MSV.FnPrologueEnd);
8518 VAArgOverflowSize =
8519 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8520 Value *CopySize = IRB.CreateAdd(
8521 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AMD64FpEndOffset), RHS: VAArgOverflowSize);
8522 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8523 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8524 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8525 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8526
8527 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8528 ID: Intrinsic::umin, LHS: CopySize,
8529 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8530 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8531 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8532 if (MS.TrackOrigins) {
8533 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8534 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
8535 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
8536 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8537 }
8538 }
8539
8540 // Instrument va_start.
8541 // Copy va_list shadow from the backup copy of the TLS contents.
8542 for (CallInst *OrigInst : VAStartInstrumentationList) {
8543 NextNodeIRBuilder IRB(OrigInst);
8544 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8545
8546 Value *RegSaveAreaPtrPtr =
8547 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 16));
8548 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8549 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8550 const Align Alignment = Align(16);
8551 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8552 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8553 Alignment, /*isStore*/ true);
8554 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8555 Size: AMD64FpEndOffset);
8556 if (MS.TrackOrigins)
8557 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
8558 SrcAlign: Alignment, Size: AMD64FpEndOffset);
8559 Value *OverflowArgAreaPtrPtr =
8560 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
8561 Value *OverflowArgAreaPtr =
8562 IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
8563 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
8564 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
8565 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8566 Alignment, /*isStore*/ true);
8567 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
8568 Idx0: AMD64FpEndOffset);
8569 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8570 Size: VAArgOverflowSize);
8571 if (MS.TrackOrigins) {
8572 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
8573 Idx0: AMD64FpEndOffset);
8574 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8575 Size: VAArgOverflowSize);
8576 }
8577 }
8578 }
8579};
8580
8581/// AArch64-specific implementation of VarArgHelper.
8582struct VarArgAArch64Helper : public VarArgHelperBase {
8583 static const unsigned kAArch64GrArgSize = 64;
8584 static const unsigned kAArch64VrArgSize = 128;
8585
8586 static const unsigned AArch64GrBegOffset = 0;
8587 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
8588 // Make VR space aligned to 16 bytes.
8589 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
8590 static const unsigned AArch64VrEndOffset =
8591 AArch64VrBegOffset + kAArch64VrArgSize;
8592 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
8593
8594 AllocaInst *VAArgTLSCopy = nullptr;
8595 Value *VAArgOverflowSize = nullptr;
8596
8597 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8598
8599 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
8600 MemorySanitizerVisitor &MSV)
8601 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/32) {}
8602
8603 // A very rough approximation of aarch64 argument classification rules.
8604 std::pair<ArgKind, uint64_t> classifyArgument(Type *T) {
8605 if (T->isIntOrPtrTy() && T->getPrimitiveSizeInBits() <= 64)
8606 return {AK_GeneralPurpose, 1};
8607 if (T->isFloatingPointTy() && T->getPrimitiveSizeInBits() <= 128)
8608 return {AK_FloatingPoint, 1};
8609
8610 if (T->isArrayTy()) {
8611 auto R = classifyArgument(T: T->getArrayElementType());
8612 R.second *= T->getScalarType()->getArrayNumElements();
8613 return R;
8614 }
8615
8616 if (const FixedVectorType *FV = dyn_cast<FixedVectorType>(Val: T)) {
8617 auto R = classifyArgument(T: FV->getScalarType());
8618 R.second *= FV->getNumElements();
8619 return R;
8620 }
8621
8622 LLVM_DEBUG(errs() << "Unknown vararg type: " << *T << "\n");
8623 return {AK_Memory, 0};
8624 }
8625
8626 // The instrumentation stores the argument shadow in a non ABI-specific
8627 // format because it does not know which argument is named (since Clang,
8628 // like x86_64 case, lowers the va_args in the frontend and this pass only
8629 // sees the low level code that deals with va_list internals).
8630 // The first seven GR registers are saved in the first 56 bytes of the
8631 // va_arg tls arra, followed by the first 8 FP/SIMD registers, and then
8632 // the remaining arguments.
8633 // Using constant offset within the va_arg TLS array allows fast copy
8634 // in the finalize instrumentation.
8635 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8636 unsigned GrOffset = AArch64GrBegOffset;
8637 unsigned VrOffset = AArch64VrBegOffset;
8638 unsigned OverflowOffset = AArch64VAEndOffset;
8639
8640 const DataLayout &DL = F.getDataLayout();
8641 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8642 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8643 auto [AK, RegNum] = classifyArgument(T: A->getType());
8644 if (AK == AK_GeneralPurpose &&
8645 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
8646 AK = AK_Memory;
8647 if (AK == AK_FloatingPoint &&
8648 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
8649 AK = AK_Memory;
8650 Value *Base;
8651 switch (AK) {
8652 case AK_GeneralPurpose:
8653 Base = getShadowPtrForVAArgument(IRB, ArgOffset: GrOffset);
8654 GrOffset += 8 * RegNum;
8655 break;
8656 case AK_FloatingPoint:
8657 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VrOffset);
8658 VrOffset += 16 * RegNum;
8659 break;
8660 case AK_Memory:
8661 // Don't count fixed arguments in the overflow area - va_start will
8662 // skip right over them.
8663 if (IsFixed)
8664 continue;
8665 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8666 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8667 unsigned BaseOffset = OverflowOffset;
8668 Base = getShadowPtrForVAArgument(IRB, ArgOffset: BaseOffset);
8669 OverflowOffset += AlignedSize;
8670 if (OverflowOffset > kParamTLSSize) {
8671 // We have no space to copy shadow there.
8672 CleanUnusedTLS(IRB, ShadowBase: Base, BaseOffset);
8673 continue;
8674 }
8675 break;
8676 }
8677 // Count Gp/Vr fixed arguments to their respective offsets, but don't
8678 // bother to actually store a shadow.
8679 if (IsFixed)
8680 continue;
8681 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8682 }
8683 Constant *OverflowSize =
8684 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AArch64VAEndOffset);
8685 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8686 }
8687
8688 // Retrieve a va_list field of 'void*' size.
8689 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8690 Value *SaveAreaPtrPtr =
8691 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8692 return IRB.CreateLoad(Ty: Type::getInt64Ty(C&: *MS.C), Ptr: SaveAreaPtrPtr);
8693 }
8694
8695 // Retrieve a va_list field of 'int' size.
8696 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
8697 Value *SaveAreaPtr =
8698 IRB.CreatePtrAdd(Ptr: VAListTag, Offset: ConstantInt::get(Ty: MS.IntptrTy, V: offset));
8699 Value *SaveArea32 = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: SaveAreaPtr);
8700 return IRB.CreateSExt(V: SaveArea32, DestTy: MS.IntptrTy);
8701 }
8702
8703 void finalizeInstrumentation() override {
8704 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8705 "finalizeInstrumentation called twice");
8706 if (!VAStartInstrumentationList.empty()) {
8707 // If there is a va_start in this function, make a backup copy of
8708 // va_arg_tls somewhere in the function entry block.
8709 IRBuilder<> IRB(MSV.FnPrologueEnd);
8710 VAArgOverflowSize =
8711 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8712 Value *CopySize = IRB.CreateAdd(
8713 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AArch64VAEndOffset), RHS: VAArgOverflowSize);
8714 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8715 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8716 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8717 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8718
8719 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8720 ID: Intrinsic::umin, LHS: CopySize,
8721 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8722 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8723 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8724 }
8725
8726 Value *GrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64GrArgSize);
8727 Value *VrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64VrArgSize);
8728
8729 // Instrument va_start, copy va_list shadow from the backup copy of
8730 // the TLS contents.
8731 for (CallInst *OrigInst : VAStartInstrumentationList) {
8732 NextNodeIRBuilder IRB(OrigInst);
8733
8734 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8735
8736 // The variadic ABI for AArch64 creates two areas to save the incoming
8737 // argument registers (one for 64-bit general register xn-x7 and another
8738 // for 128-bit FP/SIMD vn-v7).
8739 // We need then to propagate the shadow arguments on both regions
8740 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
8741 // The remaining arguments are saved on shadow for 'va::stack'.
8742 // One caveat is it requires only to propagate the non-named arguments,
8743 // however on the call site instrumentation 'all' the arguments are
8744 // saved. So to copy the shadow values from the va_arg TLS array
8745 // we need to adjust the offset for both GR and VR fields based on
8746 // the __{gr,vr}_offs value (since they are stores based on incoming
8747 // named arguments).
8748 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
8749
8750 // Read the stack pointer from the va_list.
8751 Value *StackSaveAreaPtr =
8752 IRB.CreateIntToPtr(V: getVAField64(IRB, VAListTag, offset: 0), DestTy: RegSaveAreaPtrTy);
8753
8754 // Read both the __gr_top and __gr_off and add them up.
8755 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 8);
8756 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, offset: 24);
8757
8758 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
8759 V: IRB.CreateAdd(LHS: GrTopSaveAreaPtr, RHS: GrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8760
8761 // Read both the __vr_top and __vr_off and add them up.
8762 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 16);
8763 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, offset: 28);
8764
8765 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
8766 V: IRB.CreateAdd(LHS: VrTopSaveAreaPtr, RHS: VrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8767
8768 // It does not know how many named arguments is being used and, on the
8769 // callsite all the arguments were saved. Since __gr_off is defined as
8770 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
8771 // argument by ignoring the bytes of shadow from named arguments.
8772 Value *GrRegSaveAreaShadowPtrOff =
8773 IRB.CreateAdd(LHS: GrArgSize, RHS: GrOffSaveArea);
8774
8775 Value *GrRegSaveAreaShadowPtr =
8776 MSV.getShadowOriginPtr(Addr: GrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8777 Alignment: Align(8), /*isStore*/ true)
8778 .first;
8779
8780 Value *GrSrcPtr =
8781 IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy, Offset: GrRegSaveAreaShadowPtrOff);
8782 Value *GrCopySize = IRB.CreateSub(LHS: GrArgSize, RHS: GrRegSaveAreaShadowPtrOff);
8783
8784 IRB.CreateMemCpy(Dst: GrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: GrSrcPtr, SrcAlign: Align(8),
8785 Size: GrCopySize);
8786
8787 // Again, but for FP/SIMD values.
8788 Value *VrRegSaveAreaShadowPtrOff =
8789 IRB.CreateAdd(LHS: VrArgSize, RHS: VrOffSaveArea);
8790
8791 Value *VrRegSaveAreaShadowPtr =
8792 MSV.getShadowOriginPtr(Addr: VrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8793 Alignment: Align(8), /*isStore*/ true)
8794 .first;
8795
8796 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
8797 Ptr: IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy,
8798 Offset: IRB.getInt32(C: AArch64VrBegOffset)),
8799 Offset: VrRegSaveAreaShadowPtrOff);
8800 Value *VrCopySize = IRB.CreateSub(LHS: VrArgSize, RHS: VrRegSaveAreaShadowPtrOff);
8801
8802 IRB.CreateMemCpy(Dst: VrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: VrSrcPtr, SrcAlign: Align(8),
8803 Size: VrCopySize);
8804
8805 // And finally for remaining arguments.
8806 Value *StackSaveAreaShadowPtr =
8807 MSV.getShadowOriginPtr(Addr: StackSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8808 Alignment: Align(16), /*isStore*/ true)
8809 .first;
8810
8811 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
8812 Ptr: VAArgTLSCopy, Offset: IRB.getInt32(C: AArch64VAEndOffset));
8813
8814 IRB.CreateMemCpy(Dst: StackSaveAreaShadowPtr, DstAlign: Align(16), Src: StackSrcPtr,
8815 SrcAlign: Align(16), Size: VAArgOverflowSize);
8816 }
8817 }
8818};
8819
8820/// PowerPC64-specific implementation of VarArgHelper.
8821struct VarArgPowerPC64Helper : public VarArgHelperBase {
8822 AllocaInst *VAArgTLSCopy = nullptr;
8823 Value *VAArgSize = nullptr;
8824
8825 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
8826 MemorySanitizerVisitor &MSV)
8827 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
8828
8829 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8830 // For PowerPC, we need to deal with alignment of stack arguments -
8831 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
8832 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
8833 // For that reason, we compute current offset from stack pointer (which is
8834 // always properly aligned), and offset for the first vararg, then subtract
8835 // them.
8836 unsigned VAArgBase;
8837 Triple TargetTriple(F.getParent()->getTargetTriple());
8838 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
8839 // and 32 bytes for ABIv2. This is usually determined by target
8840 // endianness, but in theory could be overridden by function attribute.
8841 if (TargetTriple.isPPC64ELFv2ABI())
8842 VAArgBase = 32;
8843 else
8844 VAArgBase = 48;
8845 unsigned VAArgOffset = VAArgBase;
8846 const DataLayout &DL = F.getDataLayout();
8847 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8848 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8849 bool IsByVal = CB.isByValArgument(ArgNo);
8850 if (IsByVal) {
8851 assert(A->getType()->isPointerTy());
8852 Type *RealTy = CB.getParamByValType(ArgNo);
8853 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8854 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(8));
8855 if (ArgAlign < 8)
8856 ArgAlign = Align(8);
8857 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8858 if (!IsFixed) {
8859 Value *Base =
8860 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8861 if (Base) {
8862 Value *AShadowPtr, *AOriginPtr;
8863 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8864 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8865 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8866
8867 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8868 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8869 }
8870 }
8871 VAArgOffset += alignTo(Size: ArgSize, A: Align(8));
8872 } else {
8873 Value *Base;
8874 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8875 Align ArgAlign = Align(8);
8876 if (A->getType()->isArrayTy()) {
8877 // Arrays are aligned to element size, except for long double
8878 // arrays, which are aligned to 8 bytes.
8879 Type *ElementTy = A->getType()->getArrayElementType();
8880 if (!ElementTy->isPPC_FP128Ty())
8881 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8882 } else if (A->getType()->isVectorTy()) {
8883 // Vectors are naturally aligned.
8884 ArgAlign = Align(ArgSize);
8885 }
8886 if (ArgAlign < 8)
8887 ArgAlign = Align(8);
8888 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8889 if (DL.isBigEndian()) {
8890 // Adjusting the shadow for argument with size < 8 to match the
8891 // placement of bits in big endian system
8892 if (ArgSize < 8)
8893 VAArgOffset += (8 - ArgSize);
8894 }
8895 if (!IsFixed) {
8896 Base =
8897 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8898 if (Base)
8899 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8900 }
8901 VAArgOffset += ArgSize;
8902 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(8));
8903 }
8904 if (IsFixed)
8905 VAArgBase = VAArgOffset;
8906 }
8907
8908 Constant *TotalVAArgSize =
8909 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8910 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8911 // a new class member i.e. it is the total size of all VarArgs.
8912 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8913 }
8914
8915 void finalizeInstrumentation() override {
8916 assert(!VAArgSize && !VAArgTLSCopy &&
8917 "finalizeInstrumentation called twice");
8918 IRBuilder<> IRB(MSV.FnPrologueEnd);
8919 VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8920 Value *CopySize = VAArgSize;
8921
8922 if (!VAStartInstrumentationList.empty()) {
8923 // If there is a va_start in this function, make a backup copy of
8924 // va_arg_tls somewhere in the function entry block.
8925
8926 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8927 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8928 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8929 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8930
8931 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8932 ID: Intrinsic::umin, LHS: CopySize,
8933 RHS: ConstantInt::get(Ty: IRB.getInt64Ty(), V: kParamTLSSize));
8934 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8935 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8936 }
8937
8938 // Instrument va_start.
8939 // Copy va_list shadow from the backup copy of the TLS contents.
8940 for (CallInst *OrigInst : VAStartInstrumentationList) {
8941 NextNodeIRBuilder IRB(OrigInst);
8942 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8943 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
8944
8945 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
8946
8947 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8948 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8949 const DataLayout &DL = F.getDataLayout();
8950 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8951 const Align Alignment = Align(IntptrSize);
8952 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8953 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8954 Alignment, /*isStore*/ true);
8955 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8956 Size: CopySize);
8957 }
8958 }
8959};
8960
8961/// PowerPC32-specific implementation of VarArgHelper.
8962struct VarArgPowerPC32Helper : public VarArgHelperBase {
8963 AllocaInst *VAArgTLSCopy = nullptr;
8964 Value *VAArgSize = nullptr;
8965
8966 VarArgPowerPC32Helper(Function &F, MemorySanitizer &MS,
8967 MemorySanitizerVisitor &MSV)
8968 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/12) {}
8969
8970 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8971 unsigned VAArgBase;
8972 // Parameter save area is 8 bytes from frame pointer in PPC32
8973 VAArgBase = 8;
8974 unsigned VAArgOffset = VAArgBase;
8975 const DataLayout &DL = F.getDataLayout();
8976 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
8977 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8978 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8979 bool IsByVal = CB.isByValArgument(ArgNo);
8980 if (IsByVal) {
8981 assert(A->getType()->isPointerTy());
8982 Type *RealTy = CB.getParamByValType(ArgNo);
8983 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8984 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
8985 if (ArgAlign < IntptrSize)
8986 ArgAlign = Align(IntptrSize);
8987 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8988 if (!IsFixed) {
8989 Value *Base =
8990 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8991 if (Base) {
8992 Value *AShadowPtr, *AOriginPtr;
8993 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8994 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8995 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8996
8997 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8998 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8999 }
9000 }
9001 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9002 } else {
9003 Value *Base;
9004 Type *ArgTy = A->getType();
9005
9006 // On PPC 32 floating point variable arguments are stored in separate
9007 // area: fp_save_area = reg_save_area + 4*8. We do not copy shaodow for
9008 // them as they will be found when checking call arguments.
9009 if (!ArgTy->isFloatingPointTy()) {
9010 uint64_t ArgSize = DL.getTypeAllocSize(Ty: ArgTy);
9011 Align ArgAlign = Align(IntptrSize);
9012 if (ArgTy->isArrayTy()) {
9013 // Arrays are aligned to element size, except for long double
9014 // arrays, which are aligned to 8 bytes.
9015 Type *ElementTy = ArgTy->getArrayElementType();
9016 if (!ElementTy->isPPC_FP128Ty())
9017 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
9018 } else if (ArgTy->isVectorTy()) {
9019 // Vectors are naturally aligned.
9020 ArgAlign = Align(ArgSize);
9021 }
9022 if (ArgAlign < IntptrSize)
9023 ArgAlign = Align(IntptrSize);
9024 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9025 if (DL.isBigEndian()) {
9026 // Adjusting the shadow for argument with size < IntptrSize to match
9027 // the placement of bits in big endian system
9028 if (ArgSize < IntptrSize)
9029 VAArgOffset += (IntptrSize - ArgSize);
9030 }
9031 if (!IsFixed) {
9032 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase,
9033 ArgSize);
9034 if (Base)
9035 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base,
9036 Align: kShadowTLSAlignment);
9037 }
9038 VAArgOffset += ArgSize;
9039 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9040 }
9041 }
9042 }
9043
9044 Constant *TotalVAArgSize =
9045 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
9046 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9047 // a new class member i.e. it is the total size of all VarArgs.
9048 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9049 }
9050
9051 void finalizeInstrumentation() override {
9052 assert(!VAArgSize && !VAArgTLSCopy &&
9053 "finalizeInstrumentation called twice");
9054 IRBuilder<> IRB(MSV.FnPrologueEnd);
9055 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9056 Value *CopySize = VAArgSize;
9057
9058 if (!VAStartInstrumentationList.empty()) {
9059 // If there is a va_start in this function, make a backup copy of
9060 // va_arg_tls somewhere in the function entry block.
9061
9062 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9063 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9064 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9065 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9066
9067 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9068 ID: Intrinsic::umin, LHS: CopySize,
9069 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9070 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9071 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9072 }
9073
9074 // Instrument va_start.
9075 // Copy va_list shadow from the backup copy of the TLS contents.
9076 for (CallInst *OrigInst : VAStartInstrumentationList) {
9077 NextNodeIRBuilder IRB(OrigInst);
9078 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9079 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9080 Value *RegSaveAreaSize = CopySize;
9081
9082 // In PPC32 va_list_tag is a struct
9083 RegSaveAreaPtrPtr =
9084 IRB.CreateAdd(LHS: RegSaveAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
9085
9086 // On PPC 32 reg_save_area can only hold 32 bytes of data
9087 RegSaveAreaSize = IRB.CreateBinaryIntrinsic(
9088 ID: Intrinsic::umin, LHS: CopySize, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
9089
9090 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
9091 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9092
9093 const DataLayout &DL = F.getDataLayout();
9094 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9095 const Align Alignment = Align(IntptrSize);
9096
9097 { // Copy reg save area
9098 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9099 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9100 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9101 Alignment, /*isStore*/ true);
9102 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy,
9103 SrcAlign: Alignment, Size: RegSaveAreaSize);
9104
9105 RegSaveAreaShadowPtr =
9106 IRB.CreatePtrToInt(V: RegSaveAreaShadowPtr, DestTy: MS.IntptrTy);
9107 Value *FPSaveArea = IRB.CreateAdd(LHS: RegSaveAreaShadowPtr,
9108 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
9109 FPSaveArea = IRB.CreateIntToPtr(V: FPSaveArea, DestTy: MS.PtrTy);
9110 // We fill fp shadow with zeroes as uninitialized fp args should have
9111 // been found during call base check
9112 IRB.CreateMemSet(Ptr: FPSaveArea, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
9113 Size: ConstantInt::get(Ty: MS.IntptrTy, V: 32), Align: Alignment);
9114 }
9115
9116 { // Copy overflow area
9117 // RegSaveAreaSize is min(CopySize, 32) -> no overflow can occur
9118 Value *OverflowAreaSize = IRB.CreateSub(LHS: CopySize, RHS: RegSaveAreaSize);
9119
9120 Value *OverflowAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9121 OverflowAreaPtrPtr =
9122 IRB.CreateAdd(LHS: OverflowAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 4));
9123 OverflowAreaPtrPtr = IRB.CreateIntToPtr(V: OverflowAreaPtrPtr, DestTy: MS.PtrTy);
9124
9125 Value *OverflowAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowAreaPtrPtr);
9126
9127 Value *OverflowAreaShadowPtr, *OverflowAreaOriginPtr;
9128 std::tie(args&: OverflowAreaShadowPtr, args&: OverflowAreaOriginPtr) =
9129 MSV.getShadowOriginPtr(Addr: OverflowAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9130 Alignment, /*isStore*/ true);
9131
9132 Value *OverflowVAArgTLSCopyPtr =
9133 IRB.CreatePtrToInt(V: VAArgTLSCopy, DestTy: MS.IntptrTy);
9134 OverflowVAArgTLSCopyPtr =
9135 IRB.CreateAdd(LHS: OverflowVAArgTLSCopyPtr, RHS: RegSaveAreaSize);
9136
9137 OverflowVAArgTLSCopyPtr =
9138 IRB.CreateIntToPtr(V: OverflowVAArgTLSCopyPtr, DestTy: MS.PtrTy);
9139 IRB.CreateMemCpy(Dst: OverflowAreaShadowPtr, DstAlign: Alignment,
9140 Src: OverflowVAArgTLSCopyPtr, SrcAlign: Alignment, Size: OverflowAreaSize);
9141 }
9142 }
9143 }
9144};
9145
9146/// SystemZ-specific implementation of VarArgHelper.
9147struct VarArgSystemZHelper : public VarArgHelperBase {
9148 static const unsigned SystemZGpOffset = 16;
9149 static const unsigned SystemZGpEndOffset = 56;
9150 static const unsigned SystemZFpOffset = 128;
9151 static const unsigned SystemZFpEndOffset = 160;
9152 static const unsigned SystemZMaxVrArgs = 8;
9153 static const unsigned SystemZRegSaveAreaSize = 160;
9154 static const unsigned SystemZOverflowOffset = 160;
9155 static const unsigned SystemZVAListTagSize = 32;
9156 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
9157 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
9158
9159 bool IsSoftFloatABI;
9160 AllocaInst *VAArgTLSCopy = nullptr;
9161 AllocaInst *VAArgTLSOriginCopy = nullptr;
9162 Value *VAArgOverflowSize = nullptr;
9163
9164 enum class ArgKind {
9165 GeneralPurpose,
9166 FloatingPoint,
9167 Vector,
9168 Memory,
9169 Indirect,
9170 };
9171
9172 enum class ShadowExtension { None, Zero, Sign };
9173
9174 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
9175 MemorySanitizerVisitor &MSV)
9176 : VarArgHelperBase(F, MS, MSV, SystemZVAListTagSize),
9177 IsSoftFloatABI(F.getFnAttribute(Kind: "use-soft-float").getValueAsBool()) {}
9178
9179 ArgKind classifyArgument(Type *T) {
9180 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
9181 // only a few possibilities of what it can be. In particular, enums, single
9182 // element structs and large types have already been taken care of.
9183
9184 // Some i128 and fp128 arguments are converted to pointers only in the
9185 // back end.
9186 if (T->isIntegerTy(BitWidth: 128) || T->isFP128Ty())
9187 return ArgKind::Indirect;
9188 if (T->isFloatingPointTy())
9189 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
9190 if (T->isIntegerTy() || T->isPointerTy())
9191 return ArgKind::GeneralPurpose;
9192 if (T->isVectorTy())
9193 return ArgKind::Vector;
9194 return ArgKind::Memory;
9195 }
9196
9197 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
9198 // ABI says: "One of the simple integer types no more than 64 bits wide.
9199 // ... If such an argument is shorter than 64 bits, replace it by a full
9200 // 64-bit integer representing the same number, using sign or zero
9201 // extension". Shadow for an integer argument has the same type as the
9202 // argument itself, so it can be sign or zero extended as well.
9203 bool ZExt = CB.paramHasAttr(ArgNo, Kind: Attribute::ZExt);
9204 bool SExt = CB.paramHasAttr(ArgNo, Kind: Attribute::SExt);
9205 if (ZExt) {
9206 assert(!SExt);
9207 return ShadowExtension::Zero;
9208 }
9209 if (SExt) {
9210 assert(!ZExt);
9211 return ShadowExtension::Sign;
9212 }
9213 return ShadowExtension::None;
9214 }
9215
9216 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9217 unsigned GpOffset = SystemZGpOffset;
9218 unsigned FpOffset = SystemZFpOffset;
9219 unsigned VrIndex = 0;
9220 unsigned OverflowOffset = SystemZOverflowOffset;
9221 const DataLayout &DL = F.getDataLayout();
9222 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9223 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9224 // SystemZABIInfo does not produce ByVal parameters.
9225 assert(!CB.isByValArgument(ArgNo));
9226 Type *T = A->getType();
9227 ArgKind AK = classifyArgument(T);
9228 if (AK == ArgKind::Indirect) {
9229 T = MS.PtrTy;
9230 AK = ArgKind::GeneralPurpose;
9231 }
9232 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
9233 AK = ArgKind::Memory;
9234 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
9235 AK = ArgKind::Memory;
9236 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
9237 AK = ArgKind::Memory;
9238 Value *ShadowBase = nullptr;
9239 Value *OriginBase = nullptr;
9240 ShadowExtension SE = ShadowExtension::None;
9241 switch (AK) {
9242 case ArgKind::GeneralPurpose: {
9243 // Always keep track of GpOffset, but store shadow only for varargs.
9244 uint64_t ArgSize = 8;
9245 if (GpOffset + ArgSize <= kParamTLSSize) {
9246 if (!IsFixed) {
9247 SE = getShadowExtension(CB, ArgNo);
9248 uint64_t GapSize = 0;
9249 if (SE == ShadowExtension::None) {
9250 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9251 assert(ArgAllocSize <= ArgSize);
9252 GapSize = ArgSize - ArgAllocSize;
9253 }
9254 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9255 if (MS.TrackOrigins)
9256 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9257 }
9258 GpOffset += ArgSize;
9259 } else {
9260 GpOffset = kParamTLSSize;
9261 }
9262 break;
9263 }
9264 case ArgKind::FloatingPoint: {
9265 // Always keep track of FpOffset, but store shadow only for varargs.
9266 uint64_t ArgSize = 8;
9267 if (FpOffset + ArgSize <= kParamTLSSize) {
9268 if (!IsFixed) {
9269 // PoP says: "A short floating-point datum requires only the
9270 // left-most 32 bit positions of a floating-point register".
9271 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
9272 // don't extend shadow and don't mind the gap.
9273 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: FpOffset);
9274 if (MS.TrackOrigins)
9275 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
9276 }
9277 FpOffset += ArgSize;
9278 } else {
9279 FpOffset = kParamTLSSize;
9280 }
9281 break;
9282 }
9283 case ArgKind::Vector: {
9284 // Keep track of VrIndex. No need to store shadow, since vector varargs
9285 // go through AK_Memory.
9286 assert(IsFixed);
9287 VrIndex++;
9288 break;
9289 }
9290 case ArgKind::Memory: {
9291 // Keep track of OverflowOffset and store shadow only for varargs.
9292 // Ignore fixed args, since we need to copy only the vararg portion of
9293 // the overflow area shadow.
9294 if (!IsFixed) {
9295 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9296 uint64_t ArgSize = alignTo(Value: ArgAllocSize, Align: 8);
9297 if (OverflowOffset + ArgSize <= kParamTLSSize) {
9298 SE = getShadowExtension(CB, ArgNo);
9299 uint64_t GapSize =
9300 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
9301 ShadowBase =
9302 getShadowAddrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9303 if (MS.TrackOrigins)
9304 OriginBase =
9305 getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9306 OverflowOffset += ArgSize;
9307 } else {
9308 OverflowOffset = kParamTLSSize;
9309 }
9310 }
9311 break;
9312 }
9313 case ArgKind::Indirect:
9314 llvm_unreachable("Indirect must be converted to GeneralPurpose");
9315 }
9316 if (ShadowBase == nullptr)
9317 continue;
9318 Value *Shadow = MSV.getShadow(V: A);
9319 if (SE != ShadowExtension::None)
9320 Shadow = MSV.CreateShadowCast(IRB, V: Shadow, dstTy: IRB.getInt64Ty(),
9321 /*Signed*/ SE == ShadowExtension::Sign);
9322 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: MS.PtrTy, Name: "_msarg_va_s");
9323 IRB.CreateStore(Val: Shadow, Ptr: ShadowBase);
9324 if (MS.TrackOrigins) {
9325 Value *Origin = MSV.getOrigin(V: A);
9326 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
9327 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
9328 Alignment: kMinOriginAlignment);
9329 }
9330 }
9331 Constant *OverflowSize = ConstantInt::get(
9332 Ty: IRB.getInt64Ty(), V: OverflowOffset - SystemZOverflowOffset);
9333 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
9334 }
9335
9336 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
9337 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
9338 V: IRB.CreateAdd(
9339 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9340 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZRegSaveAreaPtrOffset)),
9341 DestTy: MS.PtrTy);
9342 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9343 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9344 const Align Alignment = Align(8);
9345 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9346 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
9347 /*isStore*/ true);
9348 // TODO(iii): copy only fragments filled by visitCallBase()
9349 // TODO(iii): support packed-stack && !use-soft-float
9350 // For use-soft-float functions, it is enough to copy just the GPRs.
9351 unsigned RegSaveAreaSize =
9352 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
9353 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9354 Size: RegSaveAreaSize);
9355 if (MS.TrackOrigins)
9356 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
9357 SrcAlign: Alignment, Size: RegSaveAreaSize);
9358 }
9359
9360 // FIXME: This implementation limits OverflowOffset to kParamTLSSize, so we
9361 // don't know real overflow size and can't clear shadow beyond kParamTLSSize.
9362 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
9363 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
9364 V: IRB.CreateAdd(
9365 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9366 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowArgAreaPtrOffset)),
9367 DestTy: MS.PtrTy);
9368 Value *OverflowArgAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
9369 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
9370 const Align Alignment = Align(8);
9371 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
9372 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9373 Alignment, /*isStore*/ true);
9374 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
9375 Idx0: SystemZOverflowOffset);
9376 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9377 Size: VAArgOverflowSize);
9378 if (MS.TrackOrigins) {
9379 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
9380 Idx0: SystemZOverflowOffset);
9381 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9382 Size: VAArgOverflowSize);
9383 }
9384 }
9385
9386 void finalizeInstrumentation() override {
9387 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
9388 "finalizeInstrumentation called twice");
9389 if (!VAStartInstrumentationList.empty()) {
9390 // If there is a va_start in this function, make a backup copy of
9391 // va_arg_tls somewhere in the function entry block.
9392 IRBuilder<> IRB(MSV.FnPrologueEnd);
9393 VAArgOverflowSize =
9394 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
9395 Value *CopySize =
9396 IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowOffset),
9397 RHS: VAArgOverflowSize);
9398 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9399 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9400 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9401 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9402
9403 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9404 ID: Intrinsic::umin, LHS: CopySize,
9405 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9406 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9407 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9408 if (MS.TrackOrigins) {
9409 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9410 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
9411 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
9412 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9413 }
9414 }
9415
9416 // Instrument va_start.
9417 // Copy va_list shadow from the backup copy of the TLS contents.
9418 for (CallInst *OrigInst : VAStartInstrumentationList) {
9419 NextNodeIRBuilder IRB(OrigInst);
9420 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9421 copyRegSaveArea(IRB, VAListTag);
9422 copyOverflowArea(IRB, VAListTag);
9423 }
9424 }
9425};
9426
9427/// i386-specific implementation of VarArgHelper.
9428struct VarArgI386Helper : public VarArgHelperBase {
9429 AllocaInst *VAArgTLSCopy = nullptr;
9430 Value *VAArgSize = nullptr;
9431
9432 VarArgI386Helper(Function &F, MemorySanitizer &MS,
9433 MemorySanitizerVisitor &MSV)
9434 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/4) {}
9435
9436 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9437 const DataLayout &DL = F.getDataLayout();
9438 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9439 unsigned VAArgOffset = 0;
9440 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9441 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9442 bool IsByVal = CB.isByValArgument(ArgNo);
9443 if (IsByVal) {
9444 assert(A->getType()->isPointerTy());
9445 Type *RealTy = CB.getParamByValType(ArgNo);
9446 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
9447 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
9448 if (ArgAlign < IntptrSize)
9449 ArgAlign = Align(IntptrSize);
9450 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9451 if (!IsFixed) {
9452 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9453 if (Base) {
9454 Value *AShadowPtr, *AOriginPtr;
9455 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
9456 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
9457 Alignment: kShadowTLSAlignment, /*isStore*/ false);
9458
9459 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
9460 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
9461 }
9462 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9463 }
9464 } else {
9465 Value *Base;
9466 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9467 Align ArgAlign = Align(IntptrSize);
9468 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9469 if (DL.isBigEndian()) {
9470 // Adjusting the shadow for argument with size < IntptrSize to match
9471 // the placement of bits in big endian system
9472 if (ArgSize < IntptrSize)
9473 VAArgOffset += (IntptrSize - ArgSize);
9474 }
9475 if (!IsFixed) {
9476 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9477 if (Base)
9478 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9479 VAArgOffset += ArgSize;
9480 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9481 }
9482 }
9483 }
9484
9485 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9486 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9487 // a new class member i.e. it is the total size of all VarArgs.
9488 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9489 }
9490
9491 void finalizeInstrumentation() override {
9492 assert(!VAArgSize && !VAArgTLSCopy &&
9493 "finalizeInstrumentation called twice");
9494 IRBuilder<> IRB(MSV.FnPrologueEnd);
9495 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9496 Value *CopySize = VAArgSize;
9497
9498 if (!VAStartInstrumentationList.empty()) {
9499 // If there is a va_start in this function, make a backup copy of
9500 // va_arg_tls somewhere in the function entry block.
9501 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9502 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9503 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9504 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9505
9506 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9507 ID: Intrinsic::umin, LHS: CopySize,
9508 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9509 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9510 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9511 }
9512
9513 // Instrument va_start.
9514 // Copy va_list shadow from the backup copy of the TLS contents.
9515 for (CallInst *OrigInst : VAStartInstrumentationList) {
9516 NextNodeIRBuilder IRB(OrigInst);
9517 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9518 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9519 Value *RegSaveAreaPtrPtr =
9520 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9521 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9522 Value *RegSaveAreaPtr =
9523 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9524 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9525 const DataLayout &DL = F.getDataLayout();
9526 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9527 const Align Alignment = Align(IntptrSize);
9528 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9529 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9530 Alignment, /*isStore*/ true);
9531 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9532 Size: CopySize);
9533 }
9534 }
9535};
9536
9537/// Implementation of VarArgHelper that is used for ARM32, MIPS, RISCV,
9538/// LoongArch64.
9539struct VarArgGenericHelper : public VarArgHelperBase {
9540 AllocaInst *VAArgTLSCopy = nullptr;
9541 Value *VAArgSize = nullptr;
9542
9543 VarArgGenericHelper(Function &F, MemorySanitizer &MS,
9544 MemorySanitizerVisitor &MSV, const unsigned VAListTagSize)
9545 : VarArgHelperBase(F, MS, MSV, VAListTagSize) {}
9546
9547 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9548 unsigned VAArgOffset = 0;
9549 const DataLayout &DL = F.getDataLayout();
9550 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9551 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9552 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9553 if (IsFixed)
9554 continue;
9555 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9556 if (DL.isBigEndian()) {
9557 // Adjusting the shadow for argument with size < IntptrSize to match the
9558 // placement of bits in big endian system
9559 if (ArgSize < IntptrSize)
9560 VAArgOffset += (IntptrSize - ArgSize);
9561 }
9562 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9563 VAArgOffset += ArgSize;
9564 VAArgOffset = alignTo(Value: VAArgOffset, Align: IntptrSize);
9565 if (!Base)
9566 continue;
9567 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9568 }
9569
9570 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9571 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9572 // a new class member i.e. it is the total size of all VarArgs.
9573 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9574 }
9575
9576 void finalizeInstrumentation() override {
9577 assert(!VAArgSize && !VAArgTLSCopy &&
9578 "finalizeInstrumentation called twice");
9579 IRBuilder<> IRB(MSV.FnPrologueEnd);
9580 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9581 Value *CopySize = VAArgSize;
9582
9583 if (!VAStartInstrumentationList.empty()) {
9584 // If there is a va_start in this function, make a backup copy of
9585 // va_arg_tls somewhere in the function entry block.
9586 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9587 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9588 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9589 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9590
9591 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9592 ID: Intrinsic::umin, LHS: CopySize,
9593 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9594 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9595 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9596 }
9597
9598 // Instrument va_start.
9599 // Copy va_list shadow from the backup copy of the TLS contents.
9600 for (CallInst *OrigInst : VAStartInstrumentationList) {
9601 NextNodeIRBuilder IRB(OrigInst);
9602 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9603 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9604 Value *RegSaveAreaPtrPtr =
9605 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9606 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9607 Value *RegSaveAreaPtr =
9608 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9609 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9610 const DataLayout &DL = F.getDataLayout();
9611 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9612 const Align Alignment = Align(IntptrSize);
9613 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9614 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9615 Alignment, /*isStore*/ true);
9616 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9617 Size: CopySize);
9618 }
9619 }
9620};
9621
9622// ARM32, Loongarch64, MIPS and RISCV share the same calling conventions
9623// regarding VAArgs.
9624using VarArgARM32Helper = VarArgGenericHelper;
9625using VarArgRISCVHelper = VarArgGenericHelper;
9626using VarArgMIPSHelper = VarArgGenericHelper;
9627using VarArgLoongArch64Helper = VarArgGenericHelper;
9628using VarArgHexagonHelper = VarArgGenericHelper;
9629
9630/// A no-op implementation of VarArgHelper.
9631struct VarArgNoOpHelper : public VarArgHelper {
9632 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
9633 MemorySanitizerVisitor &MSV) {}
9634
9635 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
9636
9637 void visitVAStartInst(VAStartInst &I) override {}
9638
9639 void visitVACopyInst(VACopyInst &I) override {}
9640
9641 void finalizeInstrumentation() override {}
9642};
9643
9644} // end anonymous namespace
9645
9646static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
9647 MemorySanitizerVisitor &Visitor) {
9648 // VarArg handling is only implemented on AMD64. False positives are possible
9649 // on other platforms.
9650 Triple TargetTriple(Func.getParent()->getTargetTriple());
9651
9652 if (TargetTriple.getArch() == Triple::x86)
9653 return new VarArgI386Helper(Func, Msan, Visitor);
9654
9655 if (TargetTriple.getArch() == Triple::x86_64)
9656 return new VarArgAMD64Helper(Func, Msan, Visitor);
9657
9658 if (TargetTriple.isARM())
9659 return new VarArgARM32Helper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9660
9661 if (TargetTriple.isAArch64())
9662 return new VarArgAArch64Helper(Func, Msan, Visitor);
9663
9664 if (TargetTriple.isSystemZ())
9665 return new VarArgSystemZHelper(Func, Msan, Visitor);
9666
9667 // On PowerPC32 VAListTag is a struct
9668 // {char, char, i16 padding, char *, char *}
9669 if (TargetTriple.isPPC32())
9670 return new VarArgPowerPC32Helper(Func, Msan, Visitor);
9671
9672 if (TargetTriple.isPPC64())
9673 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
9674
9675 if (TargetTriple.isRISCV32())
9676 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9677
9678 if (TargetTriple.isRISCV64())
9679 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9680
9681 if (TargetTriple.isMIPS32())
9682 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9683
9684 if (TargetTriple.isMIPS64())
9685 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9686
9687 if (TargetTriple.isLoongArch64())
9688 return new VarArgLoongArch64Helper(Func, Msan, Visitor,
9689 /*VAListTagSize=*/8);
9690
9691 if (TargetTriple.getArch() == Triple::hexagon)
9692 return new VarArgHexagonHelper(Func, Msan, Visitor, /*VAListTagSize=*/12);
9693
9694 return new VarArgNoOpHelper(Func, Msan, Visitor);
9695}
9696
9697bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
9698 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
9699 return false;
9700
9701 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
9702 return false;
9703
9704 MemorySanitizerVisitor Visitor(F, *this, TLI);
9705
9706 // Clear out memory attributes.
9707 AttributeMask B;
9708 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
9709 F.removeFnAttrs(Attrs: B);
9710
9711 return Visitor.runOnFunction();
9712}
9713